eotdl 2023.7.19.post3__py3-none-any.whl → 2023.9.14__py3-none-any.whl

eotdl/curation/stac/utils.py

@@ -3,11 +3,13 @@ STAC utils
 """
 
 import pystac
+import json
 
+from os.path import dirname, join, exists
+from os import listdir
 from datetime import datetime
 from dateutil import parser
 from pandas import isna
-from numpy import nan
 from typing import Union
 
 
@@ -84,3 +86,47 @@ def get_all_children(obj: pystac.STACObject) -> list:
             children.append(item.to_dict())
 
     return children
+
+
+def cut_images(images_list: Union[list, tuple]) -> list:
+    """
+    """
+    dirnames = list()
+    images = list()
+
+    for image in images_list:
+        dir = dirname(image)
+        if dir not in dirnames:
+            dirnames.append(dir)
+            images.append(image)
+
+    return images
+
+
+def get_item_metadata(raster_path: str) -> str:
+    """
+    Get the metadata JSON file of a given directory, associated to a raster file
+
+    :param raster_path: path to the raster file
+    """
+    # Get the directory of the raster file
+    raster_dir_path = dirname(raster_path)
+    # Get the metadata JSON file
+    # Check if there is a metadata.json file in the directory
+    if 'metadata.json' in listdir(raster_dir_path):
+        metadata_json = join(raster_dir_path, 'metadata.json')
+    else:
+        # If there is no metadata.json file in the directory, check if there is
+        # a json file with the same name as the raster file
+        raster_name = raster_path.split('/')[-1]
+        raster_name = raster_name.split('.')[0]
+        metadata_json = join(raster_dir_path, f'{raster_name}.json')
+        if not exists(metadata_json):
+            # If there is no metadata.json file in the directory, return None
+            return None
+    
+    # Open the metadata.json file and return it
+    with open(metadata_json, 'r') as f:
+        metadata = json.load(f)
+    
+    return metadata

eotdl/datasets/__init__.py

@@ -1,3 +1,3 @@
-from .ingest import ingest_file, ingest_folder, ingest_q1, ingest_stac
-from .download import download_dataset
+from .ingest import ingest_dataset
+from .download import download_dataset, download_file_url
 from .retrieve import retrieve_datasets, retrieve_dataset, list_datasets

eotdl/datasets/download.py

@@ -1,13 +1,26 @@
 from ..src.repos import APIRepo
-from ..src.usecases.datasets import DownloadDataset, DownloadFile
+from ..src.usecases.datasets import DownloadDataset, DownloadFileURL
 from .retrieve import retrieve_dataset
 from ..auth import with_auth
 
 
 @with_auth
-def download_dataset(dataset, file, path=None, logger=None, user=None):
+def download_dataset(
+    dataset, file=None, path=None, logger=None, assets=False, force=False, user=None
+):
     api_repo = APIRepo()
     download = DownloadDataset(api_repo, retrieve_dataset, logger)
-    inputs = download.Inputs(dataset=dataset, file=file, path=path, user=user)
+    inputs = download.Inputs(
+        dataset=dataset, file=file, path=path, user=user, assets=assets, force=force
+    )
+    outputs = download(inputs)
+    return outputs.dst_path
+
+
+@with_auth
+def download_file_url(url, path, progress=True, logger=None, user=None):
+    api_repo = APIRepo()
+    download = DownloadFileURL(api_repo, logger, progress)
+    inputs = DownloadFileURL.Inputs(url=url, path=path, user=user)
     outputs = download(inputs)
     return outputs.dst_path

eotdl/datasets/ingest.py

@@ -1,4 +1,4 @@
-import os
+from pathlib import Path
 
 from ..src.repos import APIRepo
 from ..src.usecases.datasets import IngestFile, IngestFolder, IngestSTAC
@@ -8,28 +8,39 @@ from ..auth import with_auth
 allowed_extensions = [
     ".zip",
     ".tar",
-    ".tar.gz",
+    ".gz",
     ".csv",
     ".txt",
     ".json",
+    ".geojson",
     ".pdf",
     ".md",
     ".yml",
 ]
 
 
-def ingest_q1(dataset, stac_catalog):
-    print("hola")
-    return
+def ingest_dataset(path, f=False, d=False, logger=print):
+    path = Path(path)
+    if not path.is_dir():
+        raise Exception("Path must be a folder")
+    if "catalog.json" in [f.name for f in path.iterdir()]:
+        return ingest_stac(path / "catalog.json", logger)
+    return ingest_folder(path, f, d, logger)
 
 
 @with_auth
 def ingest_file(
-    file, dataset_id, logger=None, allowed_extensions=allowed_extensions, user=None
+    file,
+    dataset_id,
+    logger=None,
+    allowed_extensions=allowed_extensions,
+    verbose=True,
+    root=None,
+    user=None,
 ):
     api_repo = APIRepo()
-    ingest = IngestFile(api_repo, allowed_extensions, logger)
-    inputs = ingest.Inputs(file=file, dataset_id=dataset_id, user=user)
+    ingest = IngestFile(api_repo, allowed_extensions, logger, verbose)
+    inputs = ingest.Inputs(file=file, dataset_id=dataset_id, user=user, root=root)
     outputs = ingest(inputs)
     return outputs.data
 
@@ -44,9 +55,9 @@ def ingest_folder(folder, force, delete, logger=None, user=None):
 
 
 @with_auth
-def ingest_stac(stac_catalog, dataset, logger=None, user=None):
+def ingest_stac(stac_catalog, logger=None, user=None):
     api_repo = APIRepo()
-    ingest = IngestSTAC(api_repo, ingest_file, allowed_extensions)
-    inputs = ingest.Inputs(stac_catalog=stac_catalog, dataset=dataset, user=user)
+    ingest = IngestSTAC(api_repo, ingest_file, allowed_extensions, logger)
+    inputs = ingest.Inputs(stac_catalog=stac_catalog, user=user)
     outputs = ingest(inputs)
     return outputs.dataset

eotdl/datasets/retrieve.py

@@ -1,9 +1,17 @@
+import re
+
 from ..src.repos import APIRepo
 from ..src.usecases.datasets import RetrieveDatasets, RetrieveDataset
 
 
-def list_datasets():
-    return retrieve_datasets()
+def list_datasets(pattern=None):
+    datasets = retrieve_datasets()
+    if pattern:
+        regex = re.compile(rf".*{re.escape(pattern)}.*", re.IGNORECASE)
+        names = list(datasets.keys())
+        valid = [name for name in names if regex.search(name)]
+        return {name: datasets[name] for name in valid}
+    return datasets
 
 
 def retrieve_datasets():

eotdl/src/repos/APIRepo.py

@@ -2,17 +2,14 @@ import requests
 from tqdm import tqdm
 from pathlib import Path
 import os
-from concurrent.futures import ThreadPoolExecutor
-import time
-import multiprocessing
 import hashlib
 import geopandas as gpd
 
 
 class APIRepo:
-    def __init__(self, url=os.getenv("EOTDL_API_URL", "https://api.eotdl.com/")):
+    # def __init__(self, url=os.getenv("EOTDL_API_URL", "https://api.eotdl.com/")):
+    def __init__(self, url=os.getenv("EOTDL_API_URL", "http://localhost:8010/")):
         self.url = url
-        # print(self.url)
 
     def login(self):
         return requests.get(self.url + "auth/login")
@@ -24,9 +21,18 @@ class APIRepo:
         response = requests.get(self.url + "auth/logout")
         return response.json()["logout_url"]
 
+    def retrieve_credentials(self, id_token):
+        response = requests.get(
+            self.url + "auth/credentials",
+            headers={"Authorization": "Bearer " + id_token},
+        )
+        if response.status_code == 200:
+            return response.json(), None
+        return None, response.json()["detail"]
+
     def create_dataset(self, metadata, id_token):
         response = requests.post(
-            self.url + "datasets",
+            self.url + "datasets/q0",
             json=metadata,
             headers={"Authorization": "Bearer " + id_token},
         )
@@ -34,6 +40,16 @@ class APIRepo:
             return response.json(), None
         return None, response.json()["detail"]
 
+    def create_stac_dataset(self, name, id_token):
+        response = requests.post(
+            self.url + "datasets/stac",
+            json={"name": name},
+            headers={"Authorization": "Bearer " + id_token},
+        )
+        if response.status_code == 200:
+            return response.json(), None
+        return None, response.json()["detail"]
+
     def retrieve_datasets(self):
         return requests.get(self.url + "datasets").json()
 
@@ -45,21 +61,29 @@ class APIRepo:
 
     def download_file(self, dataset, dataset_id, file, id_token, path):
         url = self.url + "datasets/" + dataset_id + "/download/" + file
+        return self.download_file_url(url, path, id_token, progress=True)
+
+    def download_file_url(self, url, path, id_token, progress=False):
         headers = {"Authorization": "Bearer " + id_token}
-        path = f"{path}/{file}"
+        filename = url.split("/")[-1]
+        os.makedirs(path, exist_ok=True)
+        path = f"{path}/{filename}"
         with requests.get(url, headers=headers, stream=True) as r:
             r.raise_for_status()
             total_size = int(r.headers.get("content-length", 0))
             block_size = 1024 * 1024 * 10
-            progress_bar = tqdm(
-                total=total_size, unit="iB", unit_scale=True, unit_divisor=1024
-            )
+            if progress:
+                progress_bar = tqdm(
+                    total=total_size, unit="iB", unit_scale=True, unit_divisor=1024
+                )
             with open(path, "wb") as f:
                 for chunk in r.iter_content(block_size):
-                    progress_bar.update(len(chunk))
+                    if progress:
+                        progress_bar.update(len(chunk))
                     if chunk:
                         f.write(chunk)
-            progress_bar.close()
+            if progress:
+                progress_bar.close()
             return path
 
     def ingest_file(self, file, dataset_id, id_token, checksum=None):
@@ -75,8 +99,8 @@ class APIRepo:
 
     def ingest_file_url(self, file, dataset, id_token):
         reponse = requests.post(
-            self.url + "datasets/url",
-            json={"dataset": dataset, "url": file},
+            self.url + f"datasets/{dataset}/url",
+            json={"url": file},
             headers={"Authorization": "Bearer " + id_token},
         )
         if reponse.status_code != 200:
@@ -227,10 +251,10 @@ class APIRepo:
             return None, response.json()["detail"]
         return response.json(), None
 
-    def ingest_stac(self, stac_json, dataset, id_token):
-        reponse = requests.post(
-            self.url + "datasets/stac",
-            json={"dataset": dataset, "stac": stac_json},
+    def ingest_stac(self, stac_json, dataset_id, id_token):
+        reponse = requests.put(
+            self.url + f"datasets/stac/{dataset_id}",
+            json={"stac": stac_json},
             headers={"Authorization": "Bearer " + id_token},
         )
         if reponse.status_code != 200:

eotdl/src/repos/AuthRepo.py

@@ -7,11 +7,11 @@ import jwt
 class AuthRepo:
     def __init__(self):
         self.algorithms = ["RS256"]
-        self.home = str(Path.home())
-        self.creds_path = self.home + "/.eotdl/creds.json"
+        self.base_path = str(Path.home()) + "/.cache/eotdl/"
+        os.makedirs(self.base_path, exist_ok=True)
+        self.creds_path = self.base_path + "creds.json"
 
     def save_creds(self, data):
-        os.makedirs(self.home + "/.eotdl", exist_ok=True)
         with open(self.creds_path, "w") as f:
             json.dump(data, f)
         return self.creds_path

eotdl/src/usecases/auth/IsLogged.py

@@ -1,15 +1,17 @@
 from pydantic import BaseModel
+from typing import Union
+
 
 class IsLogged:
     def __init__(self, repo):
         self.repo = repo
 
     class Inputs(BaseModel):
-        pass 
+        pass
 
     class Outputs(BaseModel):
-        user: dict = None
+        user: Union[dict, None]
 
     def __call__(self, inputs: Inputs) -> Outputs:
         user = self.repo.load_creds()
-        return self.Outputs(user=user)
+        return self.Outputs(user=user)

eotdl/src/usecases/datasets/DownloadDataset.py

@@ -1,8 +1,11 @@
 from pydantic import BaseModel
-from ....src.utils import calculate_checksum
-from ....curation.stac import STACDataFrame
 from pathlib import Path
 import os
+from typing import Union
+from tqdm import tqdm
+
+from ....curation.stac import STACDataFrame
+from ....src.utils import calculate_checksum
 
 
 class DownloadDataset:
@@ -13,9 +16,11 @@ class DownloadDataset:
 
     class Inputs(BaseModel):
         dataset: str
-        file: str = None
-        path: str = None
+        file: Union[str, None] = None
+        path: Union[str, None] = None
         user: dict
+        assets: bool = False
+        force: bool = False
 
     class Outputs(BaseModel):
         dst_path: str
@@ -32,11 +37,20 @@ class DownloadDataset:
 
     def __call__(self, inputs: Inputs) -> Outputs:
         dataset = self.retrieve_dataset(inputs.dataset)
+        download_base_path = os.getenv(
+            "EOTDL_DOWNLOAD_PATH", str(Path.home()) + "/.cache/eotdl/datasets"
+        )
         if inputs.path is None:
-            download_path = str(Path.home()) + "/.eotdl/datasets/" + inputs.dataset
+            download_path = download_base_path + "/" + inputs.dataset
         else:
             download_path = inputs.path + "/" + inputs.dataset
         os.makedirs(download_path, exist_ok=True)
+        # check if dataset already exists
+        if os.path.exists(download_path) and not inputs.force:
+            raise Exception(
+                f"Dataset {inputs.dataset} already exists at {download_path}. To force download, use force=True or -f in the CLI."
+            )
+
         if dataset["quality"] == 0:
             if inputs.file:
                 files = [f for f in dataset["files"] if f["name"] == inputs.file]
@@ -64,6 +78,7 @@ class DownloadDataset:
                 )
             return self.Outputs(dst_path="/".join(dst_path.split("/")[:-1]))
         else:
+            self.logger("Downloading STAC metadata...")
             gdf, error = self.repo.download_stac(
                 dataset["id"],
                 inputs.user["id_token"],
@@ -74,6 +89,20 @@ class DownloadDataset:
             # df.geometry = df.geometry.apply(lambda x: Polygon() if x is None else x)
             path = inputs.path
             if path is None:
-                path = str(Path.home()) + "/.eotdl/datasets/" + dataset["name"]
+                path = download_base_path + "/" + dataset["name"]
             df.to_stac(path)
+            # download assets
+            if inputs.assets:
+                self.logger("Downloading assets...")
+                df = df.dropna(subset=["assets"])
+                for row in tqdm(df.iterrows(), total=len(df)):
+                    id = row[1]["stac_id"]
+                    # print(row[1]["links"])
+                    for k, v in row[1]["assets"].items():
+                        href = v["href"]
+                        self.repo.download_file_url(
+                            href, f"{path}/assets/{id}", inputs.user["id_token"]
+                        )
+            else:
+                self.logger("To download assets, set assets=True or -a in the CLI.")
             return self.Outputs(dst_path=path)

eotdl/src/usecases/datasets/DownloadFileURL.py

@@ -0,0 +1,22 @@
+from pydantic import BaseModel
+
+
+class DownloadFileURL:
+    def __init__(self, repo, logger, progress=True):
+        self.repo = repo
+        self.logger = logger if logger else print
+        self.progress = progress
+
+    class Inputs(BaseModel):
+        url: str
+        path: str = None
+        user: dict
+
+    class Outputs(BaseModel):
+        dst_path: str
+
+    def __call__(self, inputs: Inputs) -> Outputs:
+        dst_path = self.repo.download_file_url(
+            inputs.url, inputs.path, inputs.user["id_token"], progress=self.progress
+        )
+        return self.Outputs(dst_path=dst_path)

eotdl/src/usecases/datasets/IngestFile.py

@@ -1,20 +1,24 @@
 from pydantic import BaseModel
 import os
 import typing
+from pathlib import Path
+from glob import glob
 
 from ....src.utils import calculate_checksum
 
 
 class IngestFile:
-    def __init__(self, repo, allowed_extensions, logger):
+    def __init__(self, repo, allowed_extensions, logger, verbose=True):
         self.repo = repo
         self.allowed_extensions = allowed_extensions
         self.logger = logger if logger else print
+        self.verbose = verbose
 
     class Inputs(BaseModel):
         file: typing.Any
         dataset_id: str
         user: dict
+        root: typing.Optional[Path] = None
 
     class Outputs(BaseModel):
         data: dict
@@ -27,34 +31,50 @@ class IngestFile:
                 f"Only {', '.join(self.allowed_extensions)} files are allowed"
             )
         id_token = inputs.user["id_token"]
-        self.logger(f"Uploading file {inputs.file}...")
-        # if inputs.file.startswith("http://") or inputs.file.startswith("https://"):
-        #     data, error = self.repo.ingest_file_url(
-        #         inputs.file, inputs.metadata.name, id_token
-        #     )
-        # else:
-        self.logger("Computing checksum...")
-        checksum = calculate_checksum(inputs.file)
-        self.logger(checksum)
-        self.logger("Ingesting file...")
-        filesize = os.path.getsize(inputs.file)
-        # ingest small file
-        if filesize < 1024 * 1024 * 16:  # 16 MB
-            data, error = self.repo.ingest_file(
-                inputs.file, inputs.dataset_id, id_token, checksum
+        if self.verbose:
+            self.logger(f"Uploading file {inputs.file}...")
+        if inputs.file.startswith("http://") or inputs.file.startswith("https://"):
+            data, error = self.repo.ingest_file_url(
+                inputs.file, inputs.dataset_id, id_token
             )
-            if error:
-                raise Exception(error)
-            self.logger("Done")
-            return self.Outputs(data=data)
-        # ingest large file
-        upload_id, parts = self.repo.prepare_large_upload(
-            inputs.file, inputs.dataset_id, checksum, id_token
-        )
-        self.repo.ingest_large_dataset(inputs.file, upload_id, id_token, parts)
-        self.logger("\nCompleting upload...")
-        data, error = self.repo.complete_upload(id_token, upload_id)
+        else:
+            file_path = Path(inputs.file)
+            if not file_path.is_absolute():
+                file_path = glob(
+                    str(inputs.root) + "/**/" + os.path.basename(file_path),
+                    recursive=True,
+                )
+                if len(file_path) == 0:
+                    raise Exception(f"File {inputs.file} not found")
+                elif len(file_path) > 1:
+                    raise Exception(f"Multiple files found for {inputs.file}")
+                file_path = file_path[0]
+            if self.verbose:
+                self.logger("Computing checksum...")
+            checksum = calculate_checksum(file_path)
+            if self.verbose:
+                self.logger("Ingesting file...")
+            filesize = os.path.getsize(file_path)
+            # ingest small file
+            if filesize < 1024 * 1024 * 16:  # 16 MB
+                data, error = self.repo.ingest_file(
+                    file_path, inputs.dataset_id, id_token, checksum
+                )
+                if error:
+                    raise Exception(error)
+                if self.verbose:
+                    self.logger("Done")
+                return self.Outputs(data=data)
+            # ingest large file
+            upload_id, parts = self.repo.prepare_large_upload(
+                file_path, inputs.dataset_id, checksum, id_token
+            )
+            self.repo.ingest_large_dataset(file_path, upload_id, id_token, parts)
+            if self.verbose:
+                self.logger("\nCompleting upload...")
+            data, error = self.repo.complete_upload(id_token, upload_id)
         if error:
             raise Exception(error)
-        self.logger("Done")
+        if self.verbose:
+            self.logger("Done")
         return self.Outputs(data=data)

eotdl/src/usecases/datasets/IngestSTAC.py

@@ -1,42 +1,77 @@
 from pydantic import BaseModel
-from ....curation.stac import STACDataFrame
 import json
+from pathlib import Path
+from tqdm import tqdm
+
+from ....curation.stac import STACDataFrame
 
 
 class IngestSTAC:
-    def __init__(self, repo, ingest_file, allowed_extensions):
+    def __init__(self, repo, ingest_file, allowed_extensions, logger):
         self.repo = repo
         self.ingest_file = ingest_file
         self.allowed_extensions = allowed_extensions
+        self.logger = logger if logger else print
 
     class Inputs(BaseModel):
-        stac_catalog: str
-        dataset: str
+        stac_catalog: Path
         user: dict
 
     class Outputs(BaseModel):
         dataset: dict
 
     def __call__(self, inputs: Inputs) -> Outputs:
+        # retrieve the user's geodb credentials
+        # creds, error = self.repo.retrieve_credentials(inputs.user["id_token"])
+        # self.validate_credentials(creds)
         # load the STAC catalog as a STACsetFrame
+        self.logger("Loading STAC catalog...")
         df = STACDataFrame.from_stac_file(inputs.stac_catalog)
+        catalog = df[df["type"] == "Catalog"]
+        assert len(catalog) == 1, "STAC catalog must have exactly one root catalog"
+        dataset_name = catalog.id.iloc[0]
+        # create dataset
+        data, error = self.repo.create_stac_dataset(
+            dataset_name, inputs.user["id_token"]
+        )
+        if error:
+            data, error2 = self.repo.retrieve_dataset(dataset_name)
+            if error2:
+                raise Exception(error)
+            if data["uid"] != inputs.user["sub"]:
+                raise Exception("Dataset already exists.")
+            dataset_id = data["id"]
+            # TODO: put size to 0 or else will add up
+        else:
+            dataset_id = data["dataset_id"]
+        # TODO: check that we can ingest in geodb
         # upload all assets to EOTDL
-        for row in df.dropna(subset=["assets"]).iterrows():
+        self.logger("Uploading assets...")
+        df2 = df.dropna(subset=["assets"])
+        for row in tqdm(df2.iterrows(), total=len(df2)):
             # for asset in df.assets.dropna().values[:10]:
             try:
                 for k, v in row[1]["assets"].items():
                     data = self.ingest_file(
                         v["href"],
-                        inputs.dataset,
-                        allowed_extensions=self.allowed_extensions + [".tif", ".tiff"],
+                        dataset_id,
+                        self.logger,
+                        self.allowed_extensions + [".tif", ".tiff", ".jpg"],
+                        verbose=False,
+                        root=inputs.stac_catalog.parent.parent,  # esto será siempre así en STAC?
                     )
                     file_url = f"{self.repo.url}datasets/{data['dataset_id']}/download/{data['file_name']}"
                     df.loc[row[0], "assets"][k]["href"] = file_url
             except Exception as e:
+                self.logger(f"Error uploading asset {row[0]}: {e}")
                 break
+        # ingest the STAC catalog into geodb
+        self.logger("Ingesting STAC catalog...")
         data, error = self.repo.ingest_stac(
-            json.loads(df.to_json()), inputs.dataset, inputs.user["id_token"]
+            json.loads(df.to_json()), dataset_id, inputs.user["id_token"]
         )
         if error:
+            # TODO: delete all assets that were uploaded
             raise Exception(error)
+        self.logger("Done")
         return self.Outputs(dataset=data)

eotdl/src/usecases/datasets/RetrieveDatasets.py

@@ -10,9 +10,10 @@ class RetrieveDatasets:
         pass
 
     class Outputs(BaseModel):
-        datasets: dict
+        datasets: List[str]
 
     def __call__(self, inputs: Inputs) -> Outputs:
         data = self.repo.retrieve_datasets()
-        datasets = {d["name"]: [f["name"] for f in d["files"]] for d in data}
+        # datasets = {d["name"]: [f["name"] for f in d["files"]] for d in data}
+        datasets = [d["name"] for d in data]
         return self.Outputs(datasets=datasets)

eotdl/src/usecases/datasets/__init__.py

@@ -1,4 +1,5 @@
 from .DownloadDataset import DownloadDataset
+from .DownloadFileURL import DownloadFileURL
 from .IngestDataset import IngestDataset
 from .IngestLargeDataset import IngestLargeDataset
 from .RetrieveDataset import RetrieveDataset