eotdl 2023.6.27__py3-none-any.whl → 2023.7.19__py3-none-any.whl

eotdl/cli.py

@@ -6,6 +6,5 @@ app = typer.Typer()
 app.add_typer(auth.app, name="auth")
 app.add_typer(datasets.app, name="datasets")
 
-
 if __name__ == "__main__":
     app()

eotdl/commands/datasets.py

@@ -4,26 +4,32 @@ from pathlib import Path
 from ..datasets import (
     retrieve_datasets,
     download_dataset,
-    ingest_file,
     ingest_folder,
+    ingest_stac,
 )
 
 app = typer.Typer()
 
 
 @app.command()
-def ingest(path: Path, dataset: str):
+def ingest(
+    path: Path,
+    f: bool = typer.Option(False, "--f", help="Force ingest even if file exists"),
+    d: bool = typer.Option(False, "--d", help="Delete files not in the dataset"),
+):
     """
-    Ingest a file
+    Ingest a dataset
 
-    path: Path to folder with data (limited to 10 files, not recursive!)
-    dataset: Name of the dataset
+    path: Path to folder with the dataset
     """
     try:
-        if path.is_dir():
-            ingest_folder(path, dataset, typer.echo)
+        if not path.is_dir():
+            typer.echo("Path must be a folder")
+            return
+        if "catalog.json" in [f.name for f in path.iterdir()]:
+            ingest_stac(str(path) + "/catalog.json", typer.echo)
         else:
-            ingest_file(path, dataset, typer.echo)
+            ingest_folder(path, f, d, typer.echo)
     except Exception as e:
         typer.echo(e)
 
@@ -38,7 +44,11 @@ def list():
 
 
 @app.command()
-def get(dataset: str, file: str = None, path: str = None):
+def get(
+    dataset: str,
+    path: str = None,
+    file: str = None,
+):
     """
     Download a dataset
 

eotdl/curation/stac/__init__.py

@@ -1,4 +1,4 @@
 # from .stac import STACGenerator
 # from .utils import format_time_acquired
 # from .parsers import STACIdParser, StructuredParser, UnestructuredParser
-# from .dataframe import STACDataFrame, read_stac
+from .dataframe import STACDataFrame, read_stac

eotdl/curation/stac/dataframe.py

@@ -6,12 +6,10 @@ import pandas as pd
 import geopandas as gpd
 import pystac
 import json
-import os
-from xcube_geodb.core.geodb import GeoDBClient
 from geomet import wkt
 from os.path import join
 from os import makedirs
-
+from typing import Union
 from math import isnan
 from .utils import convert_df_geom_to_shape, get_all_children
 
@@ -27,113 +25,7 @@ class STACDataFrame(gpd.GeoDataFrame):
         """
         return read_stac(stac_file)
 
-    @classmethod
-    def from_geodb(
-        self,
-        server_url: str,
-        server_port: int | str,
-        client_id: str,
-        client_secret: str,
-        auth_aud: str,
-        collection: str,
-        database: str = None,
-    ):
-        """
-        Create a STACDataFrame from a GeoDB collection
-
-        :param server_url: GeoDB server url
-        :param server_port: GeoDB server port
-        :param client_id: GeoDB client id
-        :param client_secret: GeoDB client secret
-        :param auth_aud: GeoDB auth aud
-        :param collection: GeoDB collection
-        :param database: GeoDB database
-        """
-        geodb_client = GeoDBClient(
-            server_url=server_url,
-            server_port=server_port,
-            client_id=client_id,
-            client_secret=client_secret,
-            auth_aud=auth_aud,
-        )
-
-        data = geodb_client.get_collection(collection, database=database)
-
-        return STACDataFrame(data, crs="EPSG:4326")
-
-    def ingest(
-        self,
-        collection: str,
-        server_url: str = os.environ["SERVER_URL"],
-        server_port: int = os.environ["SERVER_PORT"],
-        client_id: str = os.environ["CLIENT_ID"],
-        client_secret: str = os.environ["CLIENT_SECRET"],
-        auth_aud: str = os.environ["AUTH_DOMAIN"],
-        database: str = None,
-    ):
-        """
-        Create a GeoDB collection from a STACDataFrame
-
-        :param collection: dataset name (GeoDB collection)
-        :param server_url: GeoDB server url
-        :param server_port: GeoDB server port
-        :param client_id: GeoDB client id
-        :param client_secret: GeoDB client secret
-        :param auth_aud: GeoDB auth aud
-        :param database: GeoDB database
-        """
-
-        geodb_client = GeoDBClient(
-            server_url=server_url,
-            server_port=server_port,
-            client_id=client_id,
-            client_secret=client_secret,
-            auth_aud=auth_aud,
-        )
-
-        # TODO: check name is unique (use eotdl-cli)
-
-        # TODO: ingest assets (only if local)
-        # TODO: rename assets in the dataframe with URLs (only if local)
-
-        # ingest to geodb
-
-        # Check if the collection already exists
-        if geodb_client.collection_exists(collection, database=database):
-            # geodb_client.drop_collection(collection, database=database)
-            raise Exception(f"Collection {collection} already exists")
-
-        # Rename the column id to stac_id, to avoid conflicts with the id column
-        self.rename(columns={"id": "stac_id"}, inplace=True)
-        # Fill the NaN with '' to avoid errors, except in the geometry column
-        copy = self.copy()
-        columns_to_fill = copy.columns.drop("geometry")
-        self[columns_to_fill] = self[columns_to_fill].fillna("")
-
-        # Create the collection if it does not exist
-        # and insert the data
-        collections = {collection: self._create_collection_structure(self.columns)}
-        geodb_client.create_collections(collections, database=database)
-
-        geodb_client.insert_into_collection(collection, database=database, values=self)
-
-        # TODO: save data in eotdl
-
-    def _create_collection_structure(self, columns: list) -> dict:
-        """
-        Create the schema structure of a GeoDB collection from a STACDataFrame
-
-        :param columns: columns of the STACDataFrame
-        """
-        stac_collection = {"crs": 4326, "properties": {}}
-
-        for column in columns:
-            if column not in ("geometry", "id"):
-                stac_collection["properties"][column] = "json"
-
-        return stac_collection
-
-    def to_stac(self):
+    def to_stac(self, path):
         """
         Create a STAC catalog and children from a STACDataFrame
         """
@@ -150,11 +42,10 @@ class STACDataFrame(gpd.GeoDataFrame):
         catalog_df = df[df["type"] == "Catalog"]
 
         if catalog_df.empty:
-            root_output_folder = "output"
-            makedirs(root_output_folder, exist_ok=True)
+            makedirs(path, exist_ok=True)
         else:
             for index, row in catalog_df.iterrows():
-                root_output_folder = row[id_column]
+                root_output_folder = path + "/" + row[id_column]
                 makedirs(root_output_folder, exist_ok=True)
                 row_json = row.to_dict()
 
@@ -228,7 +119,7 @@ class STACDataFrame(gpd.GeoDataFrame):
 
 
 def read_stac(
-    stac_file: pystac.Catalog | pystac.Collection | str,
+    stac_file: Union[pystac.Catalog, pystac.Collection, str],
     geometry_column: str = "geometry",
 ) -> STACDataFrame:
     """

eotdl/curation/stac/dataframe_bck.py

@@ -0,0 +1,253 @@
+"""
+Module for the STAC dataframe
+"""
+
+import pandas as pd
+import geopandas as gpd
+import pystac
+import json
+import os
+from xcube_geodb.core.geodb import GeoDBClient
+from geomet import wkt
+from os.path import join
+from os import makedirs
+
+from math import isnan
+from .utils import convert_df_geom_to_shape, get_all_children
+
+
+class STACDataFrame(gpd.GeoDataFrame):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    @classmethod
+    def from_stac_file(self, stac_file):
+        """
+        Create a STACDataFrame from a STAC file
+        """
+        return read_stac(stac_file)
+
+    @classmethod
+    def from_geodb(
+        self,
+        server_url: str,
+        server_port: int | str,
+        client_id: str,
+        client_secret: str,
+        auth_aud: str,
+        collection: str,
+        database: str = None,
+    ):
+        """
+        Create a STACDataFrame from a GeoDB collection
+
+        :param server_url: GeoDB server url
+        :param server_port: GeoDB server port
+        :param client_id: GeoDB client id
+        :param client_secret: GeoDB client secret
+        :param auth_aud: GeoDB auth aud
+        :param collection: GeoDB collection
+        :param database: GeoDB database
+        """
+        geodb_client = GeoDBClient(
+            server_url=server_url,
+            server_port=server_port,
+            client_id=client_id,
+            client_secret=client_secret,
+            auth_aud=auth_aud,
+        )
+
+        data = geodb_client.get_collection(collection, database=database)
+
+        return STACDataFrame(data, crs="EPSG:4326")
+
+    def ingest(
+        self,
+        collection: str,
+        server_url: str = os.environ["SERVER_URL"],
+        server_port: int = os.environ["SERVER_PORT"],
+        client_id: str = os.environ["CLIENT_ID"],
+        client_secret: str = os.environ["CLIENT_SECRET"],
+        auth_aud: str = os.environ["AUTH_DOMAIN"],
+        database: str = None,
+    ):
+        """
+        Create a GeoDB collection from a STACDataFrame
+
+        :param collection: dataset name (GeoDB collection)
+        :param server_url: GeoDB server url
+        :param server_port: GeoDB server port
+        :param client_id: GeoDB client id
+        :param client_secret: GeoDB client secret
+        :param auth_aud: GeoDB auth aud
+        :param database: GeoDB database
+        """
+
+        geodb_client = GeoDBClient(
+            server_url=server_url,
+            server_port=server_port,
+            client_id=client_id,
+            client_secret=client_secret,
+            auth_aud=auth_aud,
+        )
+
+        # TODO: check name is unique (use eotdl-cli)
+
+        # TODO: ingest assets (only if local)
+        # TODO: rename assets in the dataframe with URLs (only if local)
+
+        # ingest to geodb
+
+        # Check if the collection already exists
+        if geodb_client.collection_exists(collection, database=database):
+            # geodb_client.drop_collection(collection, database=database)
+            raise Exception(f"Collection {collection} already exists")
+
+        # Rename the column id to stac_id, to avoid conflicts with the id column
+        self.rename(columns={"id": "stac_id"}, inplace=True)
+        # Fill the NaN with '' to avoid errors, except in the geometry column
+        copy = self.copy()
+        columns_to_fill = copy.columns.drop("geometry")
+        self[columns_to_fill] = self[columns_to_fill].fillna("")
+
+        # Create the collection if it does not exist
+        # and insert the data
+        collections = {collection: self._create_collection_structure(self.columns)}
+        geodb_client.create_collections(collections, database=database)
+
+        geodb_client.insert_into_collection(collection, database=database, values=self)
+
+        # TODO: save data in eotdl
+
+    def _create_collection_structure(self, columns: list) -> dict:
+        """
+        Create the schema structure of a GeoDB collection from a STACDataFrame
+
+        :param columns: columns of the STACDataFrame
+        """
+        stac_collection = {"crs": 4326, "properties": {}}
+
+        for column in columns:
+            if column not in ("geometry", "id"):
+                stac_collection["properties"][column] = "json"
+
+        return stac_collection
+
+    def to_stac(self):
+        """
+        Create a STAC catalog and children from a STACDataFrame
+        """
+        df = self.copy()
+
+        if "id" in df.columns and "stac_id" in df.columns:
+            id_column = "stac_id"
+            stac_id_exists = True
+        else:
+            id_column = "id"
+            stac_id_exists = False
+
+        # First, create the catalog and its folder, if exists
+        catalog_df = df[df["type"] == "Catalog"]
+
+        if catalog_df.empty:
+            root_output_folder = "output"
+            makedirs(root_output_folder, exist_ok=True)
+        else:
+            for index, row in catalog_df.iterrows():
+                root_output_folder = row[id_column]
+                makedirs(root_output_folder, exist_ok=True)
+                row_json = row.to_dict()
+
+                # Curate the json row
+                row_json = self.curate_json_row(row_json, stac_id_exists)
+
+                with open(join(root_output_folder, f"catalog.json"), "w") as f:
+                    json.dump(row_json, f)
+
+        # Second, create the collections and their folders, if exist
+        collections = dict()
+        collections_df = df[df["type"] == "Collection"]
+        for index, row in collections_df.iterrows():
+            stac_output_folder = join(root_output_folder, row[id_column])
+            collections[row[id_column]] = stac_output_folder
+            makedirs(stac_output_folder, exist_ok=True)
+            row_json = row.to_dict()
+
+            # Curate the json row
+            row_json = self.curate_json_row(row_json, stac_id_exists)
+
+            with open(join(stac_output_folder, f"collection.json"), "w") as f:
+                json.dump(row_json, f)
+
+        # Then, create the items and their folders, if exist
+        features_df = df[df["type"] == "Feature"]
+        for index, row in features_df.iterrows():
+            collection = row["collection"]
+            stac_output_folder = join(collections[collection], row[id_column])
+
+            # Convert the geometry from WKT back to geojson
+            row["geometry"] = row["geometry"].wkt
+            row["geometry"] = wkt.loads(row["geometry"])
+            makedirs(stac_output_folder, exist_ok=True)
+            row_json = row.to_dict()
+
+            # Curate the json row
+            row_json = self.curate_json_row(row_json, stac_id_exists)
+
+            with open(join(stac_output_folder, f'{row_json["id"]}.json'), "w") as f:
+                json.dump(row_json, f)
+
+    def curate_json_row(self, row: dict, stac_id_exists: bool) -> dict:
+        """
+        Curate the json row of a STACDataFrame, in order to generate a valid STAC file
+
+        :param row: row of a STACDataFrame
+        :param stac_id_exists: if the stac_id column exists
+        """
+        keys_to_remove = list()
+
+        # Remove the created_at and modified_at columns, if the STACDataFrame comes from GeoDB
+        for i in "created_at", "modified_at":
+            if i in row.keys():
+                keys_to_remove.append(i)
+
+        # Rename the stac_id column to id, to avoid conflicts with the id column
+        if stac_id_exists:
+            row["id"] = row["stac_id"]
+            del row["stac_id"]
+
+        # Remove the NaN values and empty strings
+        for k, v in row.items():
+            if (isinstance(v, float) and isnan(v)) or v == "":
+                keys_to_remove.append(k)
+        for key in keys_to_remove:
+            del row[key]
+        del row["geometry"]
+
+        return row
+
+
+def read_stac(
+    stac_file: pystac.Catalog | pystac.Collection | str,
+    geometry_column: str = "geometry",
+) -> STACDataFrame:
+    """
+    Read a STAC file and return a STACDataFrame
+
+    :param stac_file: STAC file to read
+    :param geometry_column: name of the geometry column
+    """
+    if isinstance(stac_file, str):
+        stac_file = pystac.read_file(stac_file)
+    children = get_all_children(stac_file)
+
+    # Convert Dataframe to STACDataFrame
+    dataframe = pd.DataFrame(children)
+    dataframe[geometry_column] = dataframe.apply(convert_df_geom_to_shape, axis=1)
+    stac_dataframe = STACDataFrame(
+        dataframe,
+        crs="EPSG:4326",
+        geometry=gpd.GeoSeries.from_wkt(dataframe[geometry_column]),
+    )
+
+    return stac_dataframe

eotdl/datasets/__init__.py

@@ -1,3 +1,3 @@
-from .ingest import ingest_file, ingest_folder
+from .ingest import ingest_file, ingest_folder, ingest_q1, ingest_stac
 from .download import download_dataset
 from .retrieve import retrieve_datasets, retrieve_dataset, list_datasets

eotdl/datasets/ingest.py

@@ -1,7 +1,10 @@
+import os
+
 from ..src.repos import APIRepo
-from ..src.usecases.datasets import IngestFile, IngestFolder
+from ..src.usecases.datasets import IngestFile, IngestFolder, IngestSTAC
 from ..auth import with_auth
 
+
 allowed_extensions = [
     ".zip",
     ".tar",
@@ -11,51 +14,39 @@ allowed_extensions = [
     ".json",
     ".pdf",
     ".md",
+    ".yml",
 ]
 
 
+def ingest_q1(dataset, stac_catalog):
+    print("hola")
+    return
+
+
 @with_auth
-def ingest_file(file, dataset, logger=None, user=None):
+def ingest_file(
+    file, dataset_id, logger=None, allowed_extensions=allowed_extensions, user=None
+):
     api_repo = APIRepo()
     ingest = IngestFile(api_repo, allowed_extensions, logger)
-    inputs = ingest.Inputs(file=file, dataset=dataset, user=user)
+    inputs = ingest.Inputs(file=file, dataset_id=dataset_id, user=user)
     outputs = ingest(inputs)
-    return outputs.dataset
+    return outputs.data
 
 
 @with_auth
-def ingest_folder(folder, dataset, logger=None, user=None):
+def ingest_folder(folder, force, delete, logger=None, user=None):
     api_repo = APIRepo()
     ingest = IngestFolder(api_repo, ingest_file, allowed_extensions, logger)
-    inputs = ingest.Inputs(folder=folder, dataset=dataset, user=user)
+    inputs = ingest.Inputs(folder=folder, user=user, force=force, delete=delete)
     outputs = ingest(inputs)
     return outputs.dataset
 
 
-# @with_auth
-# def ingest_dataset(name, description, path, logger=None, user=None):
-#     api_repo = APIRepo()
-#     ingest = IngestDataset(
-#         api_repo,
-#     )
-#     inputs = ingest.Inputs(name=name, description=description, path=path, user=user)
-#     outputs = ingest(inputs)
-#     return outputs.dataset
-
-
-# @with_auth
-# def ingest_large_dataset(name, path, logger=None, user=None):
-#     api_repo = APIRepo()
-#     ingest = IngestLargeDataset(api_repo, logger)
-#     inputs = ingest.Inputs(name=name, path=path, user=user)
-#     outputs = ingest(inputs)
-#     return outputs.dataset
-
-
-# def ingest_q0(dataset, path):
-#     return ingest_large_dataset(dataset, path)
-
-
-# def ingest_q1(dataset, stac_catalog):
-#     print("holas")
-#     return
+@with_auth
+def ingest_stac(stac_catalog, dataset, logger=None, user=None):
+    api_repo = APIRepo()
+    ingest = IngestSTAC(api_repo, ingest_file, allowed_extensions)
+    inputs = ingest.Inputs(stac_catalog=stac_catalog, dataset=dataset, user=user)
+    outputs = ingest(inputs)
+    return outputs.dataset

eotdl/src/models/__init__.py

@@ -0,0 +1 @@
+from .metadata import Metadata

eotdl/src/models/metadata.py

@@ -0,0 +1,16 @@
+from pydantic import BaseModel, validator
+from typing import List
+
+
+class Metadata(BaseModel):
+    authors: List[str]
+    license: str
+    source: str
+    name: str
+
+    # validate source is a URL
+    @validator("source")
+    def source_is_url(cls, v):
+        if not v.startswith("http") and not v.startswith("https"):
+            raise ValueError("source must be a URL")
+        return v

eotdl/src/repos/APIRepo.py

@@ -6,11 +6,13 @@ from concurrent.futures import ThreadPoolExecutor
 import time
 import multiprocessing
 import hashlib
+import geopandas as gpd
 
 
 class APIRepo:
     def __init__(self, url=os.getenv("EOTDL_API_URL", "https://api.eotdl.com/")):
         self.url = url
+        # print(self.url)
 
     def login(self):
         return requests.get(self.url + "auth/login")
@@ -22,6 +24,16 @@ class APIRepo:
         response = requests.get(self.url + "auth/logout")
         return response.json()["logout_url"]
 
+    def create_dataset(self, metadata, id_token):
+        response = requests.post(
+            self.url + "datasets",
+            json=metadata,
+            headers={"Authorization": "Bearer " + id_token},
+        )
+        if response.status_code == 200:
+            return response.json(), None
+        return None, response.json()["detail"]
+
     def retrieve_datasets(self):
         return requests.get(self.url + "datasets").json()
 
@@ -34,12 +46,7 @@ class APIRepo:
     def download_file(self, dataset, dataset_id, file, id_token, path):
         url = self.url + "datasets/" + dataset_id + "/download/" + file
         headers = {"Authorization": "Bearer " + id_token}
-        if path is None:
-            path = str(Path.home()) + "/.eotdl/datasets/" + dataset
-            os.makedirs(path, exist_ok=True)
         path = f"{path}/{file}"
-        # if os.path.exists(path):
-        #     raise Exception("File already exists")
         with requests.get(url, headers=headers, stream=True) as r:
             r.raise_for_status()
             total_size = int(r.headers.get("content-length", 0))
@@ -55,14 +62,21 @@ class APIRepo:
             progress_bar.close()
             return path
 
-    def ingest_file(self, file, dataset, id_token, checksum):
+    def ingest_file(self, file, dataset_id, id_token, checksum=None):
         reponse = requests.post(
-            self.url + "datasets",
+            self.url + "datasets/" + dataset_id,
             files={"file": open(file, "rb")},
-            data={
-                "dataset": dataset,
-                "checksum": checksum,
-            },
+            data={"checksum": checksum} if checksum else None,
+            headers={"Authorization": "Bearer " + id_token},
+        )
+        if reponse.status_code != 200:
+            return None, reponse.json()["detail"]
+        return reponse.json(), None
+
+    def ingest_file_url(self, file, dataset, id_token):
+        reponse = requests.post(
+            self.url + "datasets/url",
+            json={"dataset": dataset, "url": file},
             headers={"Authorization": "Bearer " + id_token},
         )
         if reponse.status_code != 200:
@@ -76,11 +90,11 @@ class APIRepo:
                 break
             yield data
 
-    def prepare_large_upload(self, file, dataset, checksum, id_token):
+    def prepare_large_upload(self, file, dataset_id, checksum, id_token):
         filename = Path(file).name
         response = requests.post(
-            self.url + "datasets/uploadId",
-            json={"name": filename, "checksum": checksum, "dataset": dataset},
+            self.url + f"datasets/{dataset_id}/uploadId",
+            json={"name": filename, "checksum": checksum},
             headers={"Authorization": "Bearer " + id_token},
         )
         if response.status_code != 200:
@@ -204,63 +218,29 @@ class APIRepo:
             return None, r.json()["detail"]
         return r.json(), None
 
-    def ingest_large_dataset_parallel(
-        self,
-        path,
-        upload_id,
-        dataset_id,
-        id_token,
-        parts,
-        threads,
-    ):
-        # Create thread pool executor
-        max_workers = threads if threads > 0 else multiprocessing.cpu_count()
-        executor = ThreadPoolExecutor(max_workers=max_workers)
-
-        # Divide file into chunks and create tasks for each chunk
-        offset = 0
-        tasks = []
-        content_path = os.path.abspath(path)
-        content_size = os.stat(content_path).st_size
-        chunk_size = self.get_chunk_size(content_size)
-        total_chunks = content_size // chunk_size
-        while offset < content_size:
-            chunk_end = min(offset + chunk_size, content_size)
-            part = str(offset // chunk_size + 1)
-            if part not in parts:
-                tasks.append((offset, chunk_end, part))
-            offset = chunk_end
-
-        # Define the function that will upload each chunk
-        def upload_chunk(start, end, part):
-            # print(f"Uploading chunk {start} - {end}", part)
-            with open(content_path, "rb") as f:
-                f.seek(start)
-                chunk = f.read(end - start)
-            checksum = hashlib.md5(chunk).hexdigest()
-            response = requests.post(
-                self.url + "datasets/chunk",
-                files={"file": chunk},
-                headers={
-                    "Authorization": "Bearer " + id_token,
-                    "Upload-Id": upload_id,
-                    "Dataset-Id": dataset_id,
-                    "Checksum": checksum,
-                    "Part-Number": str(part),
-                },
-            )
-            if response.status_code != 200:
-                print(f"Failed to upload chunk {start} - {end}")
-            return response
+    def delete_file(self, dataset_id, file_name, id_token):
+        response = requests.delete(
+            self.url + "datasets/" + dataset_id + "/file/" + file_name,
+            headers={"Authorization": "Bearer " + id_token},
+        )
+        if response.status_code != 200:
+            return None, response.json()["detail"]
+        return response.json(), None
 
-        # Submit each task to the executor
-        with tqdm(total=total_chunks) as pbar:
-            futures = []
-            for task in tasks:
-                future = executor.submit(upload_chunk, *task)
-                future.add_done_callback(lambda p: pbar.update())
-                futures.append(future)
+    def ingest_stac(self, stac_json, dataset, id_token):
+        reponse = requests.post(
+            self.url + "datasets/stac",
+            json={"dataset": dataset, "stac": stac_json},
+            headers={"Authorization": "Bearer " + id_token},
+        )
+        if reponse.status_code != 200:
+            return None, reponse.json()["detail"]
+        return reponse.json(), None
 
-            # Wait for all tasks to complete
-            for future in futures:
-                future.result()
+    def download_stac(self, dataset_id, id_token):
+        url = self.url + "datasets/" + dataset_id + "/download"
+        headers = {"Authorization": "Bearer " + id_token}
+        response = requests.get(url, headers=headers)
+        if response.status_code != 200:
+            return None, response.json()["detail"]
+        return gpd.GeoDataFrame.from_features(response.json()["features"]), None

eotdl/src/usecases/datasets/DownloadDataset.py

@@ -1,5 +1,8 @@
 from pydantic import BaseModel
 from ....src.utils import calculate_checksum
+from ....curation.stac import STACDataFrame
+from pathlib import Path
+import os
 
 
 class DownloadDataset:
@@ -29,28 +32,48 @@ class DownloadDataset:
 
     def __call__(self, inputs: Inputs) -> Outputs:
         dataset = self.retrieve_dataset(inputs.dataset)
-        if inputs.file:
-            files = [f for f in dataset["files"] if f["name"] == inputs.file]
-            if not files:
-                raise Exception(f"File {inputs.file} not found")
-            if len(files) > 1:
-                raise Exception(f"Multiple files with name {inputs.file} found")
-            dst_path = self.download(
-                inputs.dataset,
+        if inputs.path is None:
+            download_path = str(Path.home()) + "/.eotdl/datasets/" + inputs.dataset
+        else:
+            download_path = inputs.path + "/" + inputs.dataset
+        os.makedirs(download_path, exist_ok=True)
+        if dataset["quality"] == 0:
+            if inputs.file:
+                files = [f for f in dataset["files"] if f["name"] == inputs.file]
+                if not files:
+                    raise Exception(f"File {inputs.file} not found")
+                if len(files) > 1:
+                    raise Exception(f"Multiple files with name {inputs.file} found")
+                dst_path = self.download(
+                    inputs.dataset,
+                    dataset["id"],
+                    inputs.file,
+                    files[0]["checksum"],
+                    download_path,
+                    inputs.user,
+                )
+                return self.Outputs(dst_path=dst_path)
+            for file in dataset["files"]:
+                dst_path = self.download(
+                    inputs.dataset,
+                    dataset["id"],
+                    file["name"],
+                    file["checksum"],
+                    download_path,
+                    inputs.user,
+                )
+            return self.Outputs(dst_path="/".join(dst_path.split("/")[:-1]))
+        else:
+            gdf, error = self.repo.download_stac(
                 dataset["id"],
-                inputs.file,
-                files[0]["checksum"],
-                inputs.path,
-                inputs.user,
+                inputs.user["id_token"],
             )
-            return self.Outputs(dst_path=dst_path)
-        for file in dataset["files"]:
-            dst_path = self.download(
-                inputs.dataset,
-                dataset["id"],
-                file["name"],
-                file["checksum"],
-                inputs.path,
-                inputs.user,
-            )
-        return self.Outputs(dst_path="/".join(dst_path.split("/")[:-1]))
+            if error:
+                raise Exception(error)
+            df = STACDataFrame(gdf)
+            # df.geometry = df.geometry.apply(lambda x: Polygon() if x is None else x)
+            path = inputs.path
+            if path is None:
+                path = str(Path.home()) + "/.eotdl/datasets/" + dataset["name"]
+            df.to_stac(path)
+            return self.Outputs(dst_path=path)

eotdl/src/usecases/datasets/IngestFile.py

@@ -13,11 +13,11 @@ class IngestFile:
 
     class Inputs(BaseModel):
         file: typing.Any
-        dataset: str = None
+        dataset_id: str
         user: dict
 
     class Outputs(BaseModel):
-        dataset: dict
+        data: dict
 
     def __call__(self, inputs: Inputs) -> Outputs:
         # validate file extension
@@ -26,31 +26,35 @@ class IngestFile:
             raise Exception(
                 f"Only {', '.join(self.allowed_extensions)} files are allowed"
             )
+        id_token = inputs.user["id_token"]
         self.logger(f"Uploading file {inputs.file}...")
+        # if inputs.file.startswith("http://") or inputs.file.startswith("https://"):
+        #     data, error = self.repo.ingest_file_url(
+        #         inputs.file, inputs.metadata.name, id_token
+        #     )
+        # else:
         self.logger("Computing checksum...")
         checksum = calculate_checksum(inputs.file)
         self.logger(checksum)
-        self.logger("Ingesting dataset...")
-        id_token = inputs.user["id_token"]
+        self.logger("Ingesting file...")
         filesize = os.path.getsize(inputs.file)
         # ingest small file
         if filesize < 1024 * 1024 * 16:  # 16 MB
             data, error = self.repo.ingest_file(
-                inputs.file, inputs.dataset, id_token, checksum
+                inputs.file, inputs.dataset_id, id_token, checksum
             )
             if error:
                 raise Exception(error)
             self.logger("Done")
-            return self.Outputs(dataset=data)
+            return self.Outputs(data=data)
         # ingest large file
         upload_id, parts = self.repo.prepare_large_upload(
-            inputs.file, inputs.dataset, checksum, id_token
+            inputs.file, inputs.dataset_id, checksum, id_token
         )
-        print(upload_id, parts)
         self.repo.ingest_large_dataset(inputs.file, upload_id, id_token, parts)
         self.logger("\nCompleting upload...")
         data, error = self.repo.complete_upload(id_token, upload_id)
         if error:
             raise Exception(error)
         self.logger("Done")
-        return self.Outputs(dataset=data)
+        return self.Outputs(data=data)

eotdl/src/usecases/datasets/IngestFolder.py

@@ -1,6 +1,8 @@
 from pydantic import BaseModel
 import os
 from pathlib import Path
+import yaml
+from ...models import Metadata
 
 
 class IngestFolder:
@@ -12,13 +14,15 @@ class IngestFolder:
 
     class Inputs(BaseModel):
         folder: Path
-        dataset: str = None
         user: dict
+        force: bool = False
+        delete: bool = False
 
     class Outputs(BaseModel):
         dataset: dict
 
     def __call__(self, inputs: Inputs) -> Outputs:
+        # validate folder
         self.logger("Uploading directory (only files, not recursive)")
         items = list(inputs.folder.glob("*"))
         filtered_items = [item for item in items if item.is_file()]
@@ -29,9 +33,66 @@ class IngestFolder:
             raise Exception("No files found in directory")
         if len(filtered_items) > 10:
             raise Exception("Too many files in directory, limited to 10")
+        if "metadata.yml" not in [item.name for item in filtered_items]:
+            raise Exception("metadata.yml not found in directory")
+        # load metadata
+        metadata = yaml.safe_load(
+            open(inputs.folder.joinpath("metadata.yml"), "r").read()
+        )
+        metadata = Metadata(**metadata)
+        # remove metadata.yml from files
+        filtered_items = [
+            item for item in filtered_items if item.name != "metadata.yml"
+        ]
+        # create dataset
+        data, error = self.repo.create_dataset(metadata.dict(), inputs.user["id_token"])
+        # dataset may already exists, but if user is owner continue ingesting files
+        current_files = []
+        if error:
+            data, error2 = self.repo.retrieve_dataset(metadata.name)
+            if error2:
+                raise Exception(error)
+            if data["uid"] != inputs.user["sub"]:
+                raise Exception("Dataset already exists.")
+            data["dataset_id"] = data["id"]
+            current_files = [item["name"] for item in data["files"]]
+            if len(current_files) > 0 and not inputs.force:
+                self.logger(
+                    "The following files already exist and will not be uploaded (use --f to force re-upload):"
+                )
+                for item in current_files:
+                    self.logger(f"{item}")
+            # TODO: delete current_files that are not in filtered_items if --delete
+            hanged_files = [
+                file
+                for file in current_files
+                if file not in [item.name for item in filtered_items]
+            ]
+            if len(hanged_files) > 0:
+                self.logger(
+                    "The following files are no longer in your dataset (use --d to delete):"
+                )
+                for item in hanged_files:
+                    self.logger(f"{item}")
+                    if inputs.delete:
+                        self.logger(f"Deleting file {item}...")
+                        _, error = self.repo.delete_file(
+                            data["dataset_id"], item, inputs.user["id_token"]
+                        )
+                        if error:
+                            self.logger(error)
+                        else:
+                            self.logger("Done")
+            filtered_items = [
+                item for item in filtered_items if item.name not in current_files
+            ]
+        dataset_id = data["dataset_id"]
+        # upload files
+        if len(filtered_items) == 0:
+            raise Exception("No files to upload")
         self.logger("The following files will be uploaded:")
         for item in filtered_items:
             self.logger(f"{item.name}")
         for item in filtered_items:
-            data = self.ingest_file(item, inputs.dataset, logger=self.logger)
+            data = self.ingest_file(item, dataset_id, logger=self.logger)
         return self.Outputs(dataset=data)

eotdl/src/usecases/datasets/IngestSTAC.py

@@ -0,0 +1,42 @@
+from pydantic import BaseModel
+from ....curation.stac import STACDataFrame
+import json
+
+
+class IngestSTAC:
+    def __init__(self, repo, ingest_file, allowed_extensions):
+        self.repo = repo
+        self.ingest_file = ingest_file
+        self.allowed_extensions = allowed_extensions
+
+    class Inputs(BaseModel):
+        stac_catalog: str
+        dataset: str
+        user: dict
+
+    class Outputs(BaseModel):
+        dataset: dict
+
+    def __call__(self, inputs: Inputs) -> Outputs:
+        # load the STAC catalog as a STACsetFrame
+        df = STACDataFrame.from_stac_file(inputs.stac_catalog)
+        # upload all assets to EOTDL
+        for row in df.dropna(subset=["assets"]).iterrows():
+            # for asset in df.assets.dropna().values[:10]:
+            try:
+                for k, v in row[1]["assets"].items():
+                    data = self.ingest_file(
+                        v["href"],
+                        inputs.dataset,
+                        allowed_extensions=self.allowed_extensions + [".tif", ".tiff"],
+                    )
+                    file_url = f"{self.repo.url}datasets/{data['dataset_id']}/download/{data['file_name']}"
+                    df.loc[row[0], "assets"][k]["href"] = file_url
+            except Exception as e:
+                break
+        data, error = self.repo.ingest_stac(
+            json.loads(df.to_json()), inputs.dataset, inputs.user["id_token"]
+        )
+        if error:
+            raise Exception(error)
+        return self.Outputs(dataset=data)

eotdl/src/usecases/datasets/__init__.py

@@ -5,3 +5,4 @@ from .RetrieveDataset import RetrieveDataset
 from .RetrieveDatasets import RetrieveDatasets
 from .IngestFile import IngestFile
 from .IngestFolder import IngestFolder
+from .IngestSTAC import IngestSTAC

{eotdl-2023.6.27.dist-info → eotdl-2023.7.19.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: eotdl
-Version: 2023.6.27
+Version: 2023.7.19
 Summary: Earth Observation Training Data Lab
 License: MIT
 Author: EarthPulse

{eotdl-2023.6.27.dist-info → eotdl-2023.7.19.dist-info}/RECORD

@@ -6,28 +6,31 @@ eotdl/access/sentinelhub/client.py,sha256=skL3EfVuBLSoOc8gbbsWuAe9UTDYSI1IdpqU8H
 eotdl/access/sentinelhub/utils.py,sha256=4xzAOoxA4A3EmBD1HX7IAbiXl9cMTfYabq0wGkFrx8A,6595
 eotdl/auth/__init__.py,sha256=gN4x9suYveP1eZr5_7IisdTVy13B-Xnm_t6N_nwoI1o,66
 eotdl/auth/main.py,sha256=q3-JcDbLJMEBSXa4Y7fofso4iK-b4QjL8QV0Ok7eqG8,857
-eotdl/cli.py,sha256=MQt1XWTrqPC-onQcAeo5Oq_pIgBQh7KdYkFPoZiJQi0,193
+eotdl/cli.py,sha256=UETgJnO1sxeLeZW1vDNLy_wtGvwXAsY7f_xjbUMZYXo,192
 eotdl/commands/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 eotdl/commands/auth.py,sha256=_Kgud59Wp8Jc1Y-o8XZRXrtd_3QfRJ4d7H8HsCg0smg,937
-eotdl/commands/datasets.py,sha256=MRl6TplRcxj3b6hAxg3z6Hoxy2fcHsLqz2p9Tmr95Dc,1298
+eotdl/commands/datasets.py,sha256=vAFNOy2AbU0FwPPO8sJSoh-VsbvaIwzErMLMYES56as,1555
 eotdl/curation/__init__.py,sha256=wdcnj-u8fwObyjF8LNebwYyUULEfp3m41X1TSjURO6w,270
 eotdl/curation/formatters.py,sha256=UllCEsWspqzoBOBy4iAz5oSh4AAMva88orFFmE5binA,2929
 eotdl/curation/metadata.py,sha256=m6mcCr7UI3Mgij3FbhQAHWetXyADJIQoL2JTUas8GoU,1552
-eotdl/curation/stac/__init__.py,sha256=mAxdEREMH7AMtpka8leqS08YOAXFM3D2IGyUjfF2vuk,201
-eotdl/curation/stac/dataframe.py,sha256=0sMc00gnYNp58ShEGvXWbBh_5QOSpvKKYA0bKC5fFL4,8495
+eotdl/curation/stac/__init__.py,sha256=YLUrLky6k1pvLciy2FrjmTauWNkJnx0L6U2ZN4VSL1c,199
+eotdl/curation/stac/dataframe.py,sha256=ewDJoFKYk6fbsOKgHAsoDDF2bnriGC06w6iVuFtXBjo,4801
+eotdl/curation/stac/dataframe_bck.py,sha256=0sMc00gnYNp58ShEGvXWbBh_5QOSpvKKYA0bKC5fFL4,8495
 eotdl/curation/stac/extensions.py,sha256=I8SRMn9lyw_0sNZ-FiJ7HYPEGozylgf58gVWBSAW6kI,5231
 eotdl/curation/stac/parsers.py,sha256=KumL2ieBt0ATGgKoGyc-AJ99zSMeLD4-tI5MF9ruYPw,1380
 eotdl/curation/stac/stac.py,sha256=COFpXQAvVBpgNf7qVv3DfIalBzdnvyVuDiEDEGgnYsA,17796
 eotdl/curation/stac/utils.py,sha256=WxC1uRY3BJJh0GFgrU3IAV4-_vwMaUyeRYwUcySfj9k,1855
-eotdl/datasets/__init__.py,sha256=rx4IN7dUrknJcaBh1NIZ7HVPaCyeMSTZ654SO8B-L8o,159
+eotdl/datasets/__init__.py,sha256=KtzMnvAoLOGtXKnTG1EdZuiEPi8TLNjRNkbeq-8iYNY,183
 eotdl/datasets/download.py,sha256=_48fOUIM2SmT-6f80sc0m3ykvo1LtEBPodSZUE_WQ7c,481
-eotdl/datasets/ingest.py,sha256=wzBkmwxFViunT3KOZAwsPs0WTz7XRTUPom0rWqCjLbA,1583
+eotdl/datasets/ingest.py,sha256=m71_yX8ZTHDbftqnBqsx1usP90HSayIqj9xDEpliYJM,1306
 eotdl/datasets/retrieve.py,sha256=sXQq5FEAAx48lf6GkaGg8zyMl4KnTRp8dU6JTUjF_RQ,534
 eotdl/hello.py,sha256=bkYLbDXy00l9-wcHDRZT9GUhsgJOHLp3N8yfYaGg6mY,74
 eotdl/src/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 eotdl/src/errors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 eotdl/src/errors/auth.py,sha256=PpnFU2DvnRo8xrM77wgskKi0tfEJ1Rhle4xv2RD1qpk,306
-eotdl/src/repos/APIRepo.py,sha256=C-SDcAxczXAY9qWJmm4kUF6QjH9-a3U9krOl4yBDBGY,10025
+eotdl/src/models/__init__.py,sha256=iFRk4hXJjI2wPT6iLhczfCMiGzJFgqBkM6d9fKiR1VU,31
+eotdl/src/models/metadata.py,sha256=L23_EziGVSDJ-WZbYUYNN22GrgbCdMGxwJhgK9uzW0U,390
+eotdl/src/repos/APIRepo.py,sha256=o-rBt_o3I7_aoNkEGTXPjX40BTUhoAUJrdIKa7etUCg,9455
 eotdl/src/repos/AuthRepo.py,sha256=5Gwj7D4MgZWdvGFgoWPQeI_OM7Gv4XWc9lkD2wMy1k4,987
 eotdl/src/repos/__init__.py,sha256=kJTtURg8RZ4GSwhFFyul-SX240r25wvLuwxIhyz9kmI,59
 eotdl/src/usecases/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -35,22 +38,23 @@ eotdl/src/usecases/auth/Auth.py,sha256=e36UNqjXSxrkM30lBmEtPd8FZr49Xt_uyTj9yieX7
 eotdl/src/usecases/auth/IsLogged.py,sha256=CTHKct0V1kCx54eQGElkzeDZoWN6CIppfDEFas6ToNI,332
 eotdl/src/usecases/auth/Logout.py,sha256=ZlCMnaSerqBr6XOCcV858qq8IMTAizx6dta-UB5qoVE,420
 eotdl/src/usecases/auth/__init__.py,sha256=w9zv66rZDKgwuzETCkvYefs2gcA1VEzvtQXfBt-KOSk,81
-eotdl/src/usecases/datasets/DownloadDataset.py,sha256=jVLlt69kI8R2ryl7TxoUJU8jBcI9_Yz5qdSYrYR_jks,1908
+eotdl/src/usecases/datasets/DownloadDataset.py,sha256=aosrRxc0mseGuDVjcNvbRc1tjw7AareDg5nuzQ7ww4s,2920
 eotdl/src/usecases/datasets/DownloadFile.py,sha256=2PxQoBepX2AsZAPWt-jbdcI0VuYLtWLK_9Y_4cQ9a5U,940
 eotdl/src/usecases/datasets/IngestDataset.py,sha256=d2H5nPXsEj-LhZpWGwNDSPs9uYNXRO2V07xsTFygQDc,953
-eotdl/src/usecases/datasets/IngestFile.py,sha256=1CLjpxymytMXnwN_zXnkW6CN5a4P9td912cXWMGp5rA,1943
-eotdl/src/usecases/datasets/IngestFolder.py,sha256=5jdcxIBQR8J8jL_4IM072NRblV-AxCo9XzYcYY2WnKA,1337
+eotdl/src/usecases/datasets/IngestFile.py,sha256=BqAtmFi_JsDVNfK3y_zvKGo0elQioV8M8AIKLkZbB74,2136
+eotdl/src/usecases/datasets/IngestFolder.py,sha256=Y9GUe0i9Tn1kYu9pn_KNf3LbKez_8Xh0pffmimeLUM0,4025
 eotdl/src/usecases/datasets/IngestLargeDataset.py,sha256=Q4sR2wyRyUsCwgFucKoNPc2SpbmX74IDXROmSwwyT4Q,1428
 eotdl/src/usecases/datasets/IngestLargeDatasetParallel.py,sha256=egcl54K_Oi0BUSJeIdoQSl8BczBQXY1kCf3SWiLuc6s,1595
+eotdl/src/usecases/datasets/IngestSTAC.py,sha256=kov4SWr7PplccyiLI7pIR5WL--OpK0hLUHLpsxdfFcA,1518
 eotdl/src/usecases/datasets/RetrieveDataset.py,sha256=rmoSLllsPx3PCnIxLhR8yiibbk6xS1Pz9yvZ0RxhsHg,421
 eotdl/src/usecases/datasets/RetrieveDatasets.py,sha256=G5cEs4N6Nq8hZwi9U8Oc5NUvwmQrZdgUUAN7m3iHKWs,455
-eotdl/src/usecases/datasets/__init__.py,sha256=7O4DFuHet8OXXVj2Kqh-1u0iTm9CZ8IVQT4N1PzgDj0,303
+eotdl/src/usecases/datasets/__init__.py,sha256=vc1eeE1lFoFw5O_OCHKkhk7NuBBWgztcuz_5hwV8roA,338
 eotdl/src/utils.py,sha256=4IB6N9jRO0chMDNJzpdnFDhC9wcFF9bO5oHq2HodcHw,479
 eotdl/tools/__init__.py,sha256=pyWj9kl-0p-KSUSZ7BV8BoSYxj7j-OfGKt9NE3qw_3Q,277
 eotdl/tools/sen12floods/__init__.py,sha256=J3McIaLi_Bp5-EIVfFWHwm0qYx7PtWydCrWwju8xFW0,215
 eotdl/tools/sen12floods/tools.py,sha256=rPOkwZw_CdecfREUPMrjYCu1jI2OBhk_8PHL2MNTdV8,8124
 eotdl/tools/stac.py,sha256=s-Js3wkqFIQwbWlr4hNTtkUgX_3Suf4A2eUUQEaE-30,636
-eotdl-2023.6.27.dist-info/entry_points.txt,sha256=s6sfxUfRrSX2IP2UbrzTFTvRCtLgw3_OKcHlOKf_5F8,39
-eotdl-2023.6.27.dist-info/WHEEL,sha256=gSF7fibx4crkLz_A-IKR6kcuq0jJ64KNCkG8_bcaEao,88
-eotdl-2023.6.27.dist-info/METADATA,sha256=DQuUeCRWodLePzJzRhuZ8XlYQ0JKOT0ub4QB1JQG7GM,739
-eotdl-2023.6.27.dist-info/RECORD,,
+eotdl-2023.7.19.dist-info/entry_points.txt,sha256=s6sfxUfRrSX2IP2UbrzTFTvRCtLgw3_OKcHlOKf_5F8,39
+eotdl-2023.7.19.dist-info/WHEEL,sha256=gSF7fibx4crkLz_A-IKR6kcuq0jJ64KNCkG8_bcaEao,88
+eotdl-2023.7.19.dist-info/METADATA,sha256=XxTgd6Vjw3W7W3QSD7FrgbevS0SfSgX-mAcm-Xa_9xk,739
+eotdl-2023.7.19.dist-info/RECORD,,