eotdl 2023.11.3.post4__py3-none-any.whl → 2024.2.15__py3-none-any.whl

eotdl/__init__.py

@@ -1 +1 @@
-__version__ = "2023.11.03-4"
+__version__ = "2024.02.15"

eotdl/access/sentinelhub/evalscripts.py

@@ -54,10 +54,6 @@ class EvalScripts:
                     };
                 }
 
-                function updateOutputMetadata(scenes, inputMetadata, outputMetadata) {
-                    outputMetadata.userData = {"scenes" : scenes};
-                }
-
                 function evaluatePixel(sample) {
                     return [sample.B01,
                             sample.B02,

eotdl/access/sentinelhub/parameters.py

@@ -51,8 +51,8 @@ class SHS1Parameters(SHParameters):
     DATA_COLLECTION = DataCollection.SENTINEL1
     RESOLUTION = 3
     EVALSCRIPT = EvalScripts.SENTINEL_1
-    FIELDS = (
-        {
+    MOSAICKING_ORDER = None
+    FIELDS = {
             "include": [
                 "id",
                 "properties.datetime",
@@ -63,8 +63,7 @@ class SHS1Parameters(SHParameters):
                 "s1:timeliness",
             ],
             "exclude": [],
-        },
-    )
+        }
     FILTER = None
 
 

eotdl/access/sentinelhub/utils.py

@@ -55,7 +55,8 @@ def imagery_from_tmp_to_dir(
     Copy imagery from tmp to output dir
     """
     downloaded_files = glob(f"{tmp_dir}/**/response.tiff")
-    assert len(downloaded_files) > 0, "No files downloaded"
+    if len(downloaded_files) == 0:
+        return
 
     makedirs(output_dir, exist_ok=True)
 

eotdl/auth/auth.py

@@ -1,43 +1,57 @@
 import time
+import os
 
 from ..repos import AuthRepo, AuthAPIRepo
 from .errors import LoginError, AuthTimeOut
 from .is_logged import is_logged
 
 
-def auth(max_t=30, interval=2):
+def auth(max_t=60, interval=2):
     user = is_logged()
     if user:
         return user
     repo, api_repo = AuthRepo(), AuthAPIRepo()
-    response = api_repo.login()
-    if response.status_code != 200:
-        raise LoginError()
-    data = response.json()
-    print("On your computer or mobile device navigate to: ", data["login_url"])
-    authenticated = False
-    t0 = time.time()
-    while not authenticated and time.time() - t0 < max_t:
-        response = api_repo.token(data["code"])
-        token_data = response.json()
-        if response.status_code == 200:
-            print("Authenticated!")
-            print("- Id Token: {}...".format(token_data["id_token"][:10]))
-            # get user credentials
-            credentials = api_repo.retrieve_credentials(token_data["id_token"])[0]
-            if credentials:
-                token_data.update(credentials)
-            # save token data in file
-            creds_path = repo.save_creds(token_data)
-            print("Saved credentials to: ", creds_path)
-            current_user = repo.decode_token(token_data)
-            authenticated = True
-            current_user["id_token"] = token_data["id_token"]
-            return current_user
-        else:
-            time.sleep(interval)
-    if not authenticated:
-        raise AuthTimeOut()
+    api_key = os.environ.get("EOTDL_API_KEY", None)
+    if api_key:
+        print("Using API Key")
+        user = {"api_key": api_key}
+        user_data, error = api_repo.retrieve_user_data(user)
+        if error:
+            raise LoginError()
+        user.update({"email": user_data["email"], "uid": user_data["uid"]})
+    else:
+        response = api_repo.login()
+        if response.status_code != 200:
+            raise LoginError()
+        data = response.json()
+        print("On your computer or mobile device navigate to: ", data["login_url"])
+        authenticated = False
+        t0 = time.time()
+        while not authenticated and time.time() - t0 < max_t:
+            response = api_repo.token(data["code"])
+            token_data = response.json()
+            if response.status_code == 200:
+                print("Authenticated!")
+                print("- Id Token: {}...".format(token_data["id_token"][:10]))
+                user = repo.decode_token(token_data)
+                authenticated = True
+                user = {
+                    "id_token": token_data["id_token"],
+                    "email": user["email"],
+                    "uid": user["sub"],
+                }
+            else:
+                time.sleep(interval)
+        if not authenticated:
+            raise AuthTimeOut()
+    # get user credentials
+    credentials = api_repo.retrieve_credentials(user)[0]
+    if credentials:
+        user.update(credentials)
+    # save token data in file
+    creds_path = repo.save_creds(user)
+    print("Saved credentials to: ", creds_path)
+    return user
 
 
 # auth decorator

eotdl/curation/stac/assets.py

@@ -3,7 +3,7 @@ Module for STAC Asset Generators
 """
 
 from os import remove, listdir
-from os.path import dirname, join, basename, abspath
+from os.path import dirname, join, basename, abspath, basename
 
 import pandas as pd
 import rasterio
@@ -78,6 +78,7 @@ class BandsAssetGenerator(STACAssetGenerator):
 
         if bands:
             with rasterio.open(raster_path, "r") as raster:
+                raster_name = basename(raster_path).split(".")[0]
                 if isinstance(bands, str):
                     bands = [bands]
                 for band in bands:
@@ -89,7 +90,7 @@ class BandsAssetGenerator(STACAssetGenerator):
                         single_band = raster.read(i + 1)
                     except IndexError:
                         single_band = raster.read(1)
-                    band_name = f"{band}.{raster_format}"
+                    band_name = f"{raster_name}_{band}.{raster_format}"
                     output_band = join(dirname(raster_path), band_name)
                     # Copy the metadata
                     metadata = raster.meta.copy()
@@ -106,45 +107,4 @@ class BandsAssetGenerator(STACAssetGenerator):
                         )
                     )
 
-            # Remove the original raster file and its metadata
-            remove(raster_path)
-            remove_raster_metadata(dirname(raster_path))
-
             return asset_list
-
-
-class ExtractedAssets(STACAssetGenerator):
-    """
-    Extracted STAC Asset Generator
-    """
-
-    type = "Extracted"
-
-    def __init__(self) -> None:
-        super().__init__()
-
-    def extract_assets(self, obj_info: pd.DataFrame):
-        """
-        Get all the files with the same extension as the image file as assets
-        """
-        asset_list = []
-        # File path
-        raster_path = obj_info["image"].values[0]
-        raster_dir = dirname(raster_path)
-        # Get the files with the same extension as the image file
-        files = [
-            f for f in listdir(raster_dir) if f.endswith(raster_path.split(".")[-1])
-        ]
-        # Instantiate pystac asset and append it to the list
-        for file in files:
-            # Get the file extension
-            raster_format = file.split(".")[-1]
-            asset_list.append(
-                pystac.Asset(
-                    href=join(raster_dir, file),
-                    title=basename(file),
-                    media_type=MEDIA_TYPES_DICT[raster_format],
-                )
-            )
-
-        return asset_list

eotdl/curation/stac/dataframe.py

@@ -22,6 +22,7 @@ class STACDataFrame(gpd.GeoDataFrame):
     """
     STACDataFrame class
     """
+
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
@@ -61,7 +62,9 @@ class STACDataFrame(gpd.GeoDataFrame):
                 # Curate the json row
                 row_json = self.curate_json_row(row_json, stac_id_exists)
 
-                with open(join(root_output_folder, "catalog.json"), "w", encoding="utf-8") as f:
+                with open(
+                    join(root_output_folder, "catalog.json"), "w", encoding="utf-8"
+                ) as f:
                     json.dump(row_json, f)
 
         # Second, create the collections and their folders, if exist
@@ -76,7 +79,9 @@ class STACDataFrame(gpd.GeoDataFrame):
             # Curate the json row
             row_json = self.curate_json_row(row_json, stac_id_exists)
 
-            with open(join(stac_output_folder, "collection.json"), "w", encoding="utf-8") as f:
+            with open(
+                join(stac_output_folder, "collection.json"), "w", encoding="utf-8"
+            ) as f:
                 json.dump(row_json, f)
 
         # Then, create the items and their folders, if exist
@@ -94,7 +99,11 @@ class STACDataFrame(gpd.GeoDataFrame):
             # Curate the json row
             row_json = self.curate_json_row(row_json, stac_id_exists)
 
-            with open(join(stac_output_folder, f'{row_json["id"]}.json'), "w", encoding="utf-8") as f:
+            with open(
+                join(stac_output_folder, f'{row_json["id"]}.json'),
+                "w",
+                encoding="utf-8",
+            ) as f:
                 json.dump(row_json, f)
 
     def curate_json_row(self, row: dict, stac_id_exists: bool) -> dict:
@@ -148,7 +157,9 @@ def read_stac(
     """
     if isinstance(stac_file, (str, Path)):
         stac_file = pystac.read_file(stac_file)  # we assume this is always a catalog
+    print(stac_file)
     stac_file.make_all_asset_hrefs_absolute()
+    print("ie")
     children = get_all_children(stac_file)
 
     # Convert Dataframe to STACDataFrame

eotdl/curation/stac/extensions/label/scaneo.py

@@ -161,8 +161,9 @@ class ScaneoLabeler(LabelExtensionObject):
         """
         label_classes = []
 
-        labels_json = glob(join(root_folder, "labels.json"))[0]
-        if exists(labels_json):
+        labels_json = glob(join(root_folder, "labels.json"))
+        if len(labels_json) > 0 and exists(labels_json[0]):
+            labels_json = labels_json[0]
             with open(labels_json, "r", encoding="utf-8") as f:
                 labels = json.load(f)
             for value in labels["labels"]:

eotdl/datasets/download.py

@@ -54,7 +54,7 @@ def download_dataset(
             dst_path = repo.download_file(
                 dataset["id"],
                 filename,
-                user["id_token"],
+                user,
                 download_path,
                 file_version,
                 progress=True,
@@ -70,7 +70,7 @@ def download_dataset(
         repo = DatasetsAPIRepo()
         gdf, error = repo.download_stac(
             dataset["id"],
-            user["id_token"],
+            user,
         )
         if error:
             raise Exception(error)
@@ -89,7 +89,7 @@ def download_dataset(
                     _, filename = href.split("/download/")
                     # will overwrite assets with same name :(
                     repo.download_file_url(
-                        href, filename, f"{download_path}/assets", user["id_token"]
+                        href, filename, f"{download_path}/assets", user
                     )
         else:
             if verbose:
@@ -101,6 +101,4 @@ def download_dataset(
 def download_file_url(url, path, progress=True, logger=print, user=None):
     repo = FilesAPIRepo()
     _, filename = url.split("/download/")
-    return repo.download_file_url(
-        url, filename, f"{path}/assets", user["id_token"], progress
-    )
+    return repo.download_file_url(url, filename, f"{path}/assets", user, progress)

eotdl/datasets/ingest.py

@@ -24,11 +24,11 @@ def retrieve_dataset(metadata, user):
     repo = DatasetsAPIRepo()
     data, error = repo.retrieve_dataset(metadata.name)
     # print(data, error)
-    if data and data["uid"] != user["sub"]:
+    if data and data["uid"] != user["uid"]:
         raise Exception("Dataset already exists.")
     if error and error == "Dataset doesn't exist":
         # create dataset
-        data, error = repo.create_dataset(metadata.dict(), user["id_token"])
+        data, error = repo.create_dataset(metadata.dict(), user)
         # print(data, error)
         if error:
             raise Exception(error)
@@ -54,11 +54,11 @@ def retrieve_stac_dataset(dataset_name, user):
     repo = DatasetsAPIRepo()
     data, error = repo.retrieve_dataset(dataset_name)
     # print(data, error)
-    if data and data["uid"] != user["sub"]:
+    if data and data["uid"] != user["uid"]:
         raise Exception("Dataset already exists.")
     if error and error == "Dataset doesn't exist":
         # create dataset
-        data, error = repo.create_stac_dataset(dataset_name, user["id_token"])
+        data, error = repo.create_stac_dataset(dataset_name, user)
         # print(data, error)
         if error:
             raise Exception(error)
@@ -82,13 +82,12 @@ def ingest_stac(stac_catalog, logger=None, user=None):
     logger("New version created, version: " + str(version))
     df2 = df.dropna(subset=["assets"])
     for row in tqdm(df2.iterrows(), total=len(df2)):
-        # for asset in df.assets.dropna().values[:10]:
         try:
             for k, v in row[1]["assets"].items():
                 data, error = files_repo.ingest_file(
                     v["href"],
                     dataset_id,
-                    user["id_token"],
+                    user,
                     calculate_checksum(v["href"]),  # is always absolute?
                     "datasets",
                     version,
@@ -102,9 +101,7 @@ def ingest_stac(stac_catalog, logger=None, user=None):
             break
     # ingest the STAC catalog into geodb
     logger("Ingesting STAC catalog...")
-    data, error = repo.ingest_stac(
-        json.loads(df.to_json()), dataset_id, user["id_token"]
-    )
+    data, error = repo.ingest_stac(json.loads(df.to_json()), dataset_id, user)
     if error:
         # TODO: delete all assets that were uploaded
         raise Exception(error)

eotdl/files/ingest.py

@@ -88,7 +88,7 @@ def generate_files_lists(
 
 
 def create_new_version(repo, dataset_or_model_id, user):
-    data, error = repo.create_version(dataset_or_model_id, user["id_token"])
+    data, error = repo.create_version(dataset_or_model_id, user)
     if error:
         raise Exception(error)
     return data["version"]
@@ -117,7 +117,7 @@ def ingest_files(repo, dataset_or_model_id, folder, verbose, logger, user, endpo
                 file["path"],
                 dataset_or_model_id,
                 file["checksum"],
-                user["id_token"],
+                user,
                 endpoint,
             )
             # print(upload_id, parts)
@@ -125,11 +125,11 @@ def ingest_files(repo, dataset_or_model_id, folder, verbose, logger, user, endpo
                 file["absolute_path"],
                 file["size"],
                 upload_id,
-                user["id_token"],
+                user,
                 parts,
                 endpoint,
             )
-            files_repo.complete_upload(user["id_token"], upload_id, version, endpoint)
+            files_repo.complete_upload(user, upload_id, version, endpoint)
     # ingest new small files in batches
     if len(upload_files) > 0:
         logger("generating batches...")
@@ -148,7 +148,7 @@ def ingest_files(repo, dataset_or_model_id, folder, verbose, logger, user, endpo
                 memory_file,
                 [f["checksum"] for f in batch],
                 dataset_or_model_id,
-                user["id_token"],
+                user,
                 endpoint,
                 version,
             )
@@ -165,7 +165,7 @@ def ingest_files(repo, dataset_or_model_id, folder, verbose, logger, user, endpo
                 batch,
                 dataset_or_model_id,
                 version,
-                user["id_token"],
+                user,
                 endpoint,
             )
             if error:

eotdl/models/download.py

@@ -68,7 +68,7 @@ def download_model(
             dst_path = repo.download_file(
                 model["id"],
                 filename,
-                user["id_token"],
+                user,
                 download_path,
                 file_version,
                 endpoint="models",
@@ -108,12 +108,3 @@ def download_model(
     #     else:
     #         logger("To download assets, set assets=True or -a in the CLI.")
     #     return Outputs(dst_path=path)
-
-
-# @with_auth
-# def download_file_url(url, path, progress=True, logger=None, user=None):
-#     api_repo = APIRepo()
-#     download = DownloadFileURL(api_repo, logger, progress)
-#     inputs = DownloadFileURL.Inputs(url=url, path=path, user=user)
-#     outputs = download(inputs)
-#     return outputs.dst_path

eotdl/models/ingest.py

@@ -21,11 +21,11 @@ def retrieve_model(metadata, user):
     repo = ModelsAPIRepo()
     data, error = repo.retrieve_model(metadata.name)
     # print(data, error)
-    if data and data["uid"] != user["sub"]:
+    if data and data["uid"] != user["uid"]:
         raise Exception("Model already exists.")
     if error and error == "Model doesn't exist":
         # create dataset
-        data, error = repo.create_model(metadata.dict(), user["id_token"])
+        data, error = repo.create_model(metadata.dict(), user)
         # print(data, error)
         if error:
             raise Exception(error)

eotdl/repos/APIRepo.py

@@ -11,3 +11,10 @@ class APIRepo:
         if response.status_code == 200:
             return response.json(), None
         return None, response.json()["detail"]
+
+    def generate_headers(self, data):
+        if "api_key" in data:
+            return {"X-API-Key": data["api_key"]}
+        if "id_token" in data:
+            return {"Authorization": "Bearer " + data["id_token"]}
+        raise Exception("Invalid headers")

eotdl/repos/AuthAPIRepo.py

@@ -16,11 +16,18 @@ class AuthAPIRepo(APIRepo):
         response = requests.get(self.url + "auth/logout")
         return response.json()["logout_url"]
 
-    def retrieve_credentials(self, id_token):
+    def retrieve_credentials(self, auth):
         response = requests.get(
             self.url + "auth/credentials",
-            headers={"Authorization": "Bearer " + id_token},
+            headers=self.generate_headers(auth),
         )
         if response.status_code == 200:
             return response.json(), None
         return None, response.json()["detail"]
+
+    def retrieve_user_data(self, auth):
+        response = requests.get(
+            self.url + "auth/me",
+            headers=self.generate_headers(auth),
+        )
+        return self.format_response(response)

eotdl/repos/AuthRepo.py

@@ -20,9 +20,13 @@ class AuthRepo:
         if os.path.exists(self.creds_path):
             with open(self.creds_path, "r") as f:
                 creds = json.load(f)
-            user = self.decode_token(creds)
-            user["id_token"] = creds["id_token"]
-            return user
+            if not "id_token" in creds and not "api_key" in creds:
+                return None
+            if "api_key" in creds and creds["api_key"] != os.getenv(
+                "EOTDL_API_KEY", None
+            ):
+                return None
+            return creds
         return None
 
     def decode_token(self, token_data):