eotdl 2023.6.14.post10__tar.gz → 2023.7.19__tar.gz

{eotdl-2023.6.14.post10 → eotdl-2023.7.19}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: eotdl
-Version: 2023.6.14.post10
+Version: 2023.7.19
 Summary: Earth Observation Training Data Lab
 License: MIT
 Author: EarthPulse

{eotdl-2023.6.14.post10 → eotdl-2023.7.19}/eotdl/cli.py

@@ -6,6 +6,5 @@ app = typer.Typer()
 app.add_typer(auth.app, name="auth")
 app.add_typer(datasets.app, name="datasets")
 
-
 if __name__ == "__main__":
     app()

eotdl-2023.7.19/eotdl/commands/datasets.py

@@ -0,0 +1,67 @@
+import typer
+from pathlib import Path
+
+from ..datasets import (
+    retrieve_datasets,
+    download_dataset,
+    ingest_folder,
+    ingest_stac,
+)
+
+app = typer.Typer()
+
+
+@app.command()
+def ingest(
+    path: Path,
+    f: bool = typer.Option(False, "--f", help="Force ingest even if file exists"),
+    d: bool = typer.Option(False, "--d", help="Delete files not in the dataset"),
+):
+    """
+    Ingest a dataset
+
+    path: Path to folder with the dataset
+    """
+    try:
+        if not path.is_dir():
+            typer.echo("Path must be a folder")
+            return
+        if "catalog.json" in [f.name for f in path.iterdir()]:
+            ingest_stac(str(path) + "/catalog.json", typer.echo)
+        else:
+            ingest_folder(path, f, d, typer.echo)
+    except Exception as e:
+        typer.echo(e)
+
+
+@app.command()
+def list():
+    """
+    List all datasets and files
+    """
+    datasets = retrieve_datasets()
+    typer.echo(datasets)
+
+
+@app.command()
+def get(
+    dataset: str,
+    path: str = None,
+    file: str = None,
+):
+    """
+    Download a dataset
+
+    dataset: Name of the dataset
+    file: Name of the file to download (optional, if not provided, the whole dataset will be downloaded)
+    path: Path to download the dataset to (optional, if not provided, the dataset will be downloaded to ~/.eotdl/datasets)
+    """
+    try:
+        dst_path = download_dataset(dataset, file, path, typer.echo)
+        typer.echo(f"Data available at {dst_path}")
+    except Exception as e:
+        typer.echo(e)
+
+
+if __name__ == "__main__":
+    app()

{eotdl-2023.6.14.post10 → eotdl-2023.7.19}/eotdl/curation/stac/__init__.py

@@ -1,4 +1,4 @@
 # from .stac import STACGenerator
 # from .utils import format_time_acquired
 # from .parsers import STACIdParser, StructuredParser, UnestructuredParser
-# from .dataframe import STACDataFrame, read_stac
+from .dataframe import STACDataFrame, read_stac

eotdl-2023.7.19/eotdl/curation/stac/dataframe.py

@@ -0,0 +1,144 @@
+"""
+Module for the STAC dataframe
+"""
+
+import pandas as pd
+import geopandas as gpd
+import pystac
+import json
+from geomet import wkt
+from os.path import join
+from os import makedirs
+from typing import Union
+from math import isnan
+from .utils import convert_df_geom_to_shape, get_all_children
+
+
+class STACDataFrame(gpd.GeoDataFrame):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    @classmethod
+    def from_stac_file(self, stac_file):
+        """
+        Create a STACDataFrame from a STAC file
+        """
+        return read_stac(stac_file)
+
+    def to_stac(self, path):
+        """
+        Create a STAC catalog and children from a STACDataFrame
+        """
+        df = self.copy()
+
+        if "id" in df.columns and "stac_id" in df.columns:
+            id_column = "stac_id"
+            stac_id_exists = True
+        else:
+            id_column = "id"
+            stac_id_exists = False
+
+        # First, create the catalog and its folder, if exists
+        catalog_df = df[df["type"] == "Catalog"]
+
+        if catalog_df.empty:
+            makedirs(path, exist_ok=True)
+        else:
+            for index, row in catalog_df.iterrows():
+                root_output_folder = path + "/" + row[id_column]
+                makedirs(root_output_folder, exist_ok=True)
+                row_json = row.to_dict()
+
+                # Curate the json row
+                row_json = self.curate_json_row(row_json, stac_id_exists)
+
+                with open(join(root_output_folder, f"catalog.json"), "w") as f:
+                    json.dump(row_json, f)
+
+        # Second, create the collections and their folders, if exist
+        collections = dict()
+        collections_df = df[df["type"] == "Collection"]
+        for index, row in collections_df.iterrows():
+            stac_output_folder = join(root_output_folder, row[id_column])
+            collections[row[id_column]] = stac_output_folder
+            makedirs(stac_output_folder, exist_ok=True)
+            row_json = row.to_dict()
+
+            # Curate the json row
+            row_json = self.curate_json_row(row_json, stac_id_exists)
+
+            with open(join(stac_output_folder, f"collection.json"), "w") as f:
+                json.dump(row_json, f)
+
+        # Then, create the items and their folders, if exist
+        features_df = df[df["type"] == "Feature"]
+        for index, row in features_df.iterrows():
+            collection = row["collection"]
+            stac_output_folder = join(collections[collection], row[id_column])
+
+            # Convert the geometry from WKT back to geojson
+            row["geometry"] = row["geometry"].wkt
+            row["geometry"] = wkt.loads(row["geometry"])
+            makedirs(stac_output_folder, exist_ok=True)
+            row_json = row.to_dict()
+
+            # Curate the json row
+            row_json = self.curate_json_row(row_json, stac_id_exists)
+
+            with open(join(stac_output_folder, f'{row_json["id"]}.json'), "w") as f:
+                json.dump(row_json, f)
+
+    def curate_json_row(self, row: dict, stac_id_exists: bool) -> dict:
+        """
+        Curate the json row of a STACDataFrame, in order to generate a valid STAC file
+
+        :param row: row of a STACDataFrame
+        :param stac_id_exists: if the stac_id column exists
+        """
+        keys_to_remove = list()
+
+        # Remove the created_at and modified_at columns, if the STACDataFrame comes from GeoDB
+        for i in "created_at", "modified_at":
+            if i in row.keys():
+                keys_to_remove.append(i)
+
+        # Rename the stac_id column to id, to avoid conflicts with the id column
+        if stac_id_exists:
+            row["id"] = row["stac_id"]
+            del row["stac_id"]
+
+        # Remove the NaN values and empty strings
+        for k, v in row.items():
+            if (isinstance(v, float) and isnan(v)) or v == "":
+                keys_to_remove.append(k)
+        for key in keys_to_remove:
+            del row[key]
+        del row["geometry"]
+
+        return row
+
+
+def read_stac(
+    stac_file: Union[pystac.Catalog, pystac.Collection, str],
+    geometry_column: str = "geometry",
+) -> STACDataFrame:
+    """
+    Read a STAC file and return a STACDataFrame
+
+    :param stac_file: STAC file to read
+    :param geometry_column: name of the geometry column
+    """
+    if isinstance(stac_file, str):
+        stac_file = pystac.read_file(stac_file)
+    children = get_all_children(stac_file)
+
+    # Convert Dataframe to STACDataFrame
+    dataframe = pd.DataFrame(children)
+    dataframe[geometry_column] = dataframe.apply(convert_df_geom_to_shape, axis=1)
+    stac_dataframe = STACDataFrame(
+        dataframe,
+        crs="EPSG:4326",
+        geometry=gpd.GeoSeries.from_wkt(dataframe[geometry_column]),
+    )
+
+    return stac_dataframe

eotdl-2023.7.19/eotdl/datasets/__init__.py

@@ -0,0 +1,3 @@
+from .ingest import ingest_file, ingest_folder, ingest_q1, ingest_stac
+from .download import download_dataset
+from .retrieve import retrieve_datasets, retrieve_dataset, list_datasets

eotdl-2023.7.19/eotdl/datasets/download.py

@@ -0,0 +1,13 @@
+from ..src.repos import APIRepo
+from ..src.usecases.datasets import DownloadDataset, DownloadFile
+from .retrieve import retrieve_dataset
+from ..auth import with_auth
+
+
+@with_auth
+def download_dataset(dataset, file, path=None, logger=None, user=None):
+    api_repo = APIRepo()
+    download = DownloadDataset(api_repo, retrieve_dataset, logger)
+    inputs = download.Inputs(dataset=dataset, file=file, path=path, user=user)
+    outputs = download(inputs)
+    return outputs.dst_path

eotdl-2023.7.19/eotdl/datasets/ingest.py

@@ -0,0 +1,52 @@
+import os
+
+from ..src.repos import APIRepo
+from ..src.usecases.datasets import IngestFile, IngestFolder, IngestSTAC
+from ..auth import with_auth
+
+
+allowed_extensions = [
+    ".zip",
+    ".tar",
+    ".tar.gz",
+    ".csv",
+    ".txt",
+    ".json",
+    ".pdf",
+    ".md",
+    ".yml",
+]
+
+
+def ingest_q1(dataset, stac_catalog):
+    print("hola")
+    return
+
+
+@with_auth
+def ingest_file(
+    file, dataset_id, logger=None, allowed_extensions=allowed_extensions, user=None
+):
+    api_repo = APIRepo()
+    ingest = IngestFile(api_repo, allowed_extensions, logger)
+    inputs = ingest.Inputs(file=file, dataset_id=dataset_id, user=user)
+    outputs = ingest(inputs)
+    return outputs.data
+
+
+@with_auth
+def ingest_folder(folder, force, delete, logger=None, user=None):
+    api_repo = APIRepo()
+    ingest = IngestFolder(api_repo, ingest_file, allowed_extensions, logger)
+    inputs = ingest.Inputs(folder=folder, user=user, force=force, delete=delete)
+    outputs = ingest(inputs)
+    return outputs.dataset
+
+
+@with_auth
+def ingest_stac(stac_catalog, dataset, logger=None, user=None):
+    api_repo = APIRepo()
+    ingest = IngestSTAC(api_repo, ingest_file, allowed_extensions)
+    inputs = ingest.Inputs(stac_catalog=stac_catalog, dataset=dataset, user=user)
+    outputs = ingest(inputs)
+    return outputs.dataset

eotdl-2023.7.19/eotdl/src/models/__init__.py

@@ -0,0 +1 @@
+from .metadata import Metadata

eotdl-2023.7.19/eotdl/src/models/metadata.py

@@ -0,0 +1,16 @@
+from pydantic import BaseModel, validator
+from typing import List
+
+
+class Metadata(BaseModel):
+    authors: List[str]
+    license: str
+    source: str
+    name: str
+
+    # validate source is a URL
+    @validator("source")
+    def source_is_url(cls, v):
+        if not v.startswith("http") and not v.startswith("https"):
+            raise ValueError("source must be a URL")
+        return v

{eotdl-2023.6.14.post10 → eotdl-2023.7.19}/eotdl/src/repos/APIRepo.py

@@ -6,11 +6,13 @@ from concurrent.futures import ThreadPoolExecutor
 import time
 import multiprocessing
 import hashlib
+import geopandas as gpd
 
 
 class APIRepo:
     def __init__(self, url=os.getenv("EOTDL_API_URL", "https://api.eotdl.com/")):
         self.url = url
+        # print(self.url)
 
     def login(self):
         return requests.get(self.url + "auth/login")
@@ -22,6 +24,16 @@ class APIRepo:
         response = requests.get(self.url + "auth/logout")
         return response.json()["logout_url"]
 
+    def create_dataset(self, metadata, id_token):
+        response = requests.post(
+            self.url + "datasets",
+            json=metadata,
+            headers={"Authorization": "Bearer " + id_token},
+        )
+        if response.status_code == 200:
+            return response.json(), None
+        return None, response.json()["detail"]
+
     def retrieve_datasets(self):
         return requests.get(self.url + "datasets").json()
 
@@ -31,12 +43,10 @@ class APIRepo:
             return response.json(), None
         return None, response.json()["detail"]
 
-    def download_dataset(self, dataset_id, id_token, path):
-        url = self.url + "datasets/" + dataset_id + "/download"
+    def download_file(self, dataset, dataset_id, file, id_token, path):
+        url = self.url + "datasets/" + dataset_id + "/download/" + file
         headers = {"Authorization": "Bearer " + id_token}
-        if path is None:
-            path = str(Path.home()) + "/.eotdl/datasets"
-            os.makedirs(path, exist_ok=True)
+        path = f"{path}/{file}"
         with requests.get(url, headers=headers, stream=True) as r:
             r.raise_for_status()
             total_size = int(r.headers.get("content-length", 0))
@@ -44,10 +54,6 @@ class APIRepo:
             progress_bar = tqdm(
                 total=total_size, unit="iB", unit_scale=True, unit_divisor=1024
             )
-            filename = r.headers.get("content-disposition").split("filename=")[1][1:-1]
-            path = f"{path}/{filename}"
-            if os.path.exists(path):
-                raise Exception("File already exists")
             with open(path, "wb") as f:
                 for chunk in r.iter_content(block_size):
                     progress_bar.update(len(chunk))
@@ -56,6 +62,27 @@ class APIRepo:
             progress_bar.close()
             return path
 
+    def ingest_file(self, file, dataset_id, id_token, checksum=None):
+        reponse = requests.post(
+            self.url + "datasets/" + dataset_id,
+            files={"file": open(file, "rb")},
+            data={"checksum": checksum} if checksum else None,
+            headers={"Authorization": "Bearer " + id_token},
+        )
+        if reponse.status_code != 200:
+            return None, reponse.json()["detail"]
+        return reponse.json(), None
+
+    def ingest_file_url(self, file, dataset, id_token):
+        reponse = requests.post(
+            self.url + "datasets/url",
+            json={"dataset": dataset, "url": file},
+            headers={"Authorization": "Bearer " + id_token},
+        )
+        if reponse.status_code != 200:
+            return None, reponse.json()["detail"]
+        return reponse.json(), None
+
     def read_in_chunks(self, file_object, CHUNK_SIZE):
         while True:
             data = file_object.read(CHUNK_SIZE)
@@ -63,18 +90,21 @@ class APIRepo:
                 break
             yield data
 
-    def prepare_large_upload(self, name, id_token, checksum):
-        url = self.url + "datasets/chunk?name=" + name + "&checksum=" + checksum
-        response = requests.get(url, headers={"Authorization": "Bearer " + id_token})
+    def prepare_large_upload(self, file, dataset_id, checksum, id_token):
+        filename = Path(file).name
+        response = requests.post(
+            self.url + f"datasets/{dataset_id}/uploadId",
+            json={"name": filename, "checksum": checksum},
+            headers={"Authorization": "Bearer " + id_token},
+        )
         if response.status_code != 200:
             raise Exception(response.json()["detail"])
         data = response.json()
-        dataset_id, upload_id, parts = (
-            data["dataset_id"],
+        upload_id, parts = (
             data["upload_id"],
             data["parts"] if "parts" in data else [],
         )
-        return dataset_id, upload_id, parts
+        return upload_id, parts
 
     def get_chunk_size(self, content_size):
         # adapt chunk size to content size to avoid S3 limits (10000 parts, 500MB per part, 5TB per object)
@@ -85,37 +115,31 @@ class APIRepo:
             chunk_size = 1024 * 1024 * 500  # 0.5 GB (up to 5 TB, 10000 parts)
         return chunk_size
 
-    def ingest_large_dataset(self, path, upload_id, dataset_id, id_token, parts):
-        content_path = os.path.abspath(path)
+    def ingest_large_dataset(self, file, upload_id, id_token, parts):
+        content_path = os.path.abspath(file)
         content_size = os.stat(content_path).st_size
         chunk_size = self.get_chunk_size(content_size)
         total_chunks = content_size // chunk_size
-        url = self.url + "datasets/chunk"
-        headers = {
-            "Authorization": "Bearer " + id_token,
-            "Upload-Id": upload_id,
-            "Dataset-Id": dataset_id,
-        }
         # upload chunks sequentially
         pbar = tqdm(
             self.read_in_chunks(open(content_path, "rb"), chunk_size),
             total=total_chunks,
         )
         index = 0
-        parts_checkusms = []
         for chunk in pbar:
             part = index // chunk_size + 1
             offset = index + len(chunk)
             index = offset
             if part not in parts:
-                headers["Part-Number"] = str(part)
                 checksum = hashlib.md5(chunk).hexdigest()
-                parts_checkusms.append(checksum)
-                headers["Checksum"] = checksum
-                file = {"file": chunk}
-                r = requests.post(url, files=file, headers=headers)
-                if r.status_code != 200:
-                    return None, r.json()["detail"]
+                response = requests.post(
+                    self.url + "datasets/chunk/" + upload_id,
+                    files={"file": chunk},
+                    data={"part_number": part, "checksum": checksum},
+                    headers={"Authorization": "Bearer " + id_token},
+                )
+                if response.status_code != 200:
+                    raise Exception(response.json()["detail"])
             pbar.set_description(
                 "{:.2f}/{:.2f} MB".format(
                     offset / 1024 / 1024, content_size / 1024 / 1024
@@ -124,16 +148,10 @@ class APIRepo:
         pbar.close()
         return
 
-    def complete_upload(self, name, id_token, upload_id, dataset_id, checksum):
-        url = self.url + "datasets/complete"
+    def complete_upload(self, id_token, upload_id):
         r = requests.post(
-            url,
-            json={"name": name, "checksum": checksum},
-            headers={
-                "Authorization": "Bearer " + id_token,
-                "Upload-Id": upload_id,
-                "Dataset-Id": dataset_id,
-            },
+            self.url + "datasets/complete/" + upload_id,
+            headers={"Authorization": "Bearer " + id_token},
         )
         if r.status_code != 200:
             return None, r.json()["detail"]
@@ -200,63 +218,29 @@ class APIRepo:
             return None, r.json()["detail"]
         return r.json(), None
 
-    def ingest_large_dataset_parallel(
-        self,
-        path,
-        upload_id,
-        dataset_id,
-        id_token,
-        parts,
-        threads,
-    ):
-        # Create thread pool executor
-        max_workers = threads if threads > 0 else multiprocessing.cpu_count()
-        executor = ThreadPoolExecutor(max_workers=max_workers)
-
-        # Divide file into chunks and create tasks for each chunk
-        offset = 0
-        tasks = []
-        content_path = os.path.abspath(path)
-        content_size = os.stat(content_path).st_size
-        chunk_size = self.get_chunk_size(content_size)
-        total_chunks = content_size // chunk_size
-        while offset < content_size:
-            chunk_end = min(offset + chunk_size, content_size)
-            part = str(offset // chunk_size + 1)
-            if part not in parts:
-                tasks.append((offset, chunk_end, part))
-            offset = chunk_end
-
-        # Define the function that will upload each chunk
-        def upload_chunk(start, end, part):
-            # print(f"Uploading chunk {start} - {end}", part)
-            with open(content_path, "rb") as f:
-                f.seek(start)
-                chunk = f.read(end - start)
-            checksum = hashlib.md5(chunk).hexdigest()
-            response = requests.post(
-                self.url + "datasets/chunk",
-                files={"file": chunk},
-                headers={
-                    "Authorization": "Bearer " + id_token,
-                    "Upload-Id": upload_id,
-                    "Dataset-Id": dataset_id,
-                    "Checksum": checksum,
-                    "Part-Number": str(part),
-                },
-            )
-            if response.status_code != 200:
-                print(f"Failed to upload chunk {start} - {end}")
-            return response
+    def delete_file(self, dataset_id, file_name, id_token):
+        response = requests.delete(
+            self.url + "datasets/" + dataset_id + "/file/" + file_name,
+            headers={"Authorization": "Bearer " + id_token},
+        )
+        if response.status_code != 200:
+            return None, response.json()["detail"]
+        return response.json(), None
 
-        # Submit each task to the executor
-        with tqdm(total=total_chunks) as pbar:
-            futures = []
-            for task in tasks:
-                future = executor.submit(upload_chunk, *task)
-                future.add_done_callback(lambda p: pbar.update())
-                futures.append(future)
+    def ingest_stac(self, stac_json, dataset, id_token):
+        reponse = requests.post(
+            self.url + "datasets/stac",
+            json={"dataset": dataset, "stac": stac_json},
+            headers={"Authorization": "Bearer " + id_token},
+        )
+        if reponse.status_code != 200:
+            return None, reponse.json()["detail"]
+        return reponse.json(), None
 
-            # Wait for all tasks to complete
-            for future in futures:
-                future.result()
+    def download_stac(self, dataset_id, id_token):
+        url = self.url + "datasets/" + dataset_id + "/download"
+        headers = {"Authorization": "Bearer " + id_token}
+        response = requests.get(url, headers=headers)
+        if response.status_code != 200:
+            return None, response.json()["detail"]
+        return gpd.GeoDataFrame.from_features(response.json()["features"]), None

eotdl-2023.7.19/eotdl/src/usecases/datasets/DownloadDataset.py

@@ -0,0 +1,79 @@
+from pydantic import BaseModel
+from ....src.utils import calculate_checksum
+from ....curation.stac import STACDataFrame
+from pathlib import Path
+import os
+
+
+class DownloadDataset:
+    def __init__(self, repo, retrieve_dataset, logger):
+        self.repo = repo
+        self.retrieve_dataset = retrieve_dataset
+        self.logger = logger if logger else print
+
+    class Inputs(BaseModel):
+        dataset: str
+        file: str = None
+        path: str = None
+        user: dict
+
+    class Outputs(BaseModel):
+        dst_path: str
+
+    def download(self, dataset, dataset_id, file, checksum, path, user):
+        self.logger(f"Downloading {file}")
+        dst_path = self.repo.download_file(
+            dataset, dataset_id, file, user["id_token"], path
+        )
+        if calculate_checksum(dst_path) != checksum:
+            self.logger(f"Checksum for {file} does not match")
+        self.logger(f"Done")
+        return dst_path
+
+    def __call__(self, inputs: Inputs) -> Outputs:
+        dataset = self.retrieve_dataset(inputs.dataset)
+        if inputs.path is None:
+            download_path = str(Path.home()) + "/.eotdl/datasets/" + inputs.dataset
+        else:
+            download_path = inputs.path + "/" + inputs.dataset
+        os.makedirs(download_path, exist_ok=True)
+        if dataset["quality"] == 0:
+            if inputs.file:
+                files = [f for f in dataset["files"] if f["name"] == inputs.file]
+                if not files:
+                    raise Exception(f"File {inputs.file} not found")
+                if len(files) > 1:
+                    raise Exception(f"Multiple files with name {inputs.file} found")
+                dst_path = self.download(
+                    inputs.dataset,
+                    dataset["id"],
+                    inputs.file,
+                    files[0]["checksum"],
+                    download_path,
+                    inputs.user,
+                )
+                return self.Outputs(dst_path=dst_path)
+            for file in dataset["files"]:
+                dst_path = self.download(
+                    inputs.dataset,
+                    dataset["id"],
+                    file["name"],
+                    file["checksum"],
+                    download_path,
+                    inputs.user,
+                )
+            return self.Outputs(dst_path="/".join(dst_path.split("/")[:-1]))
+        else:
+            gdf, error = self.repo.download_stac(
+                dataset["id"],
+                inputs.user["id_token"],
+            )
+            if error:
+                raise Exception(error)
+            df = STACDataFrame(gdf)
+            # df.geometry = df.geometry.apply(lambda x: Polygon() if x is None else x)
+            path = inputs.path
+            if path is None:
+                path = str(Path.home()) + "/.eotdl/datasets/" + dataset["name"]
+            df.to_stac(path)
+            return self.Outputs(dst_path=path)