eotdl 2024.5.2__py3-none-any.whl → 2024.6.13__py3-none-any.whl

eotdl/__init__.py

@@ -1 +1 @@
-__version__ = "2024.05.02"
+__version__ = "2024.06.13"

eotdl/models/download.py

@@ -5,8 +5,9 @@ from tqdm import tqdm
 from ..auth import with_auth
 from .retrieve import retrieve_model, retrieve_model_files
 from ..shared import calculate_checksum
-from ..repos import FilesAPIRepo
+from ..repos import FilesAPIRepo, ModelsAPIRepo
 from .metadata import generate_metadata
+from ..curation.stac import STACDataFrame
 
 
 @with_auth
@@ -46,20 +47,6 @@ def download_model(
     if model["quality"] == 0:
         if file:
             raise NotImplementedError("Downloading a specific file is not implemented")
-            # files = [f for f in model["files"] if f["name"] == file]
-            # if not files:
-            #     raise Exception(f"File {file} not found")
-            # if len(files) > 1:
-            #     raise Exception(f"Multiple files with name {file} found")
-            # dst_path = download(
-            #     model,
-            #     model["id"],
-            #     file,
-            #     files[0]["checksum"],
-            #     download_path,
-            #     user,
-            # )
-            # return Outputs(dst_path=dst_path)
         model_files = retrieve_model_files(model["id"], version)
         repo = FilesAPIRepo()
         for file in tqdm(model_files, disable=verbose, unit="file"):
@@ -74,41 +61,38 @@ def download_model(
                 file_version,
                 endpoint="models",
             )
-            # if calculate_checksum(dst_path) != checksum:
-            #     logger(f"Checksum for {file} does not match")
+            if verbose:
+                logger("Generating README.md ...")
+            generate_metadata(download_path, model)
     else:
-        raise NotImplementedError("Downloading a STAC model is not implemented")
-    #     logger("Downloading STAC metadata...")
-    #     gdf, error = repo.download_stac(
-    #         model["id"],
-    #         user["id_token"],
-    #     )
-    #     if error:
-    #         raise Exception(error)
-    #     df = STACDataFrame(gdf)
-    #     # df.geometry = df.geometry.apply(lambda x: Polygon() if x is None else x)
-    #     path = path
-    #     if path is None:
-    #         path = download_base_path + "/" + model["name"]
-    #     df.to_stac(path)
-    #     # download assets
-    #     if assets:
-    #         logger("Downloading assets...")
-    #         df = df.dropna(subset=["assets"])
-    #         for row in tqdm(df.iterrows(), total=len(df)):
-    #             id = row[1]["stac_id"]
-    #             # print(row[1]["links"])
-    #             for k, v in row[1]["assets"].items():
-    #                 href = v["href"]
-    #                 repo.download_file_url(
-    #                     href, f"{path}/assets/{id}", user["id_token"]
-    #                 )
-    #     else:
-    #         logger("To download assets, set assets=True or -a in the CLI.")
-    #     return Outputs(dst_path=path)
-    if verbose:
-        logger("Generating README.md ...")
-    generate_metadata(download_path, model)
+        if verbose:
+            logger("Downloading STAC metadata...")
+        repo = ModelsAPIRepo()
+        gdf, error = repo.download_stac(
+            model["id"],
+            user,
+        )
+        if error:
+            raise Exception(error)
+        df = STACDataFrame(gdf)
+        # df.geometry = df.geometry.apply(lambda x: Polygon() if x is None else x)
+        df.to_stac(download_path)
+        # download assets
+        if assets:
+            if verbose:
+                logger("Downloading assets...")
+            repo = FilesAPIRepo()
+            df = df.dropna(subset=["assets"])
+            for row in tqdm(df.iterrows(), total=len(df)):
+                for k, v in row[1]["assets"].items():
+                    href = v["href"]
+                    _, filename = href.split("/download/")
+                    # will overwrite assets with same name :(
+                    repo.download_file_url(
+                        href, filename, f"{download_path}/assets", user
+                    )
+        else:
+            logger("To download assets, set assets=True or -a in the CLI.")
     if verbose:
         logger("Done")
     return download_path

eotdl/models/ingest.py

@@ -2,13 +2,16 @@ from pathlib import Path
 import yaml
 import frontmatter
 import markdown
+from tqdm import tqdm
+import json
 
 from ..auth import with_auth
 from .metadata import Metadata, generate_metadata
-from ..repos import ModelsAPIRepo
+from ..repos import ModelsAPIRepo, FilesAPIRepo
 from ..shared import calculate_checksum
-from ..files import ingest_files
+from ..files import ingest_files, create_new_version
 from .update import update_model
+from ..curation.stac import STACDataFrame
 
 
 def ingest_model(
@@ -17,8 +20,8 @@ def ingest_model(
     path = Path(path)
     if not path.is_dir():
         raise Exception("Path must be a folder")
-    # if "catalog.json" in [f.name for f in path.iterdir()]:
-    #     return ingest_stac(path / "catalog.json", logger)
+    if "catalog.json" in [f.name for f in path.iterdir()]:
+        return ingest_stac(path / "catalog.json", logger)
     return ingest_folder(path, verbose, logger, force_metadata_update, sync_metadata)
 
 
@@ -101,3 +104,64 @@ def check_metadata(
             generate_metadata(str(folder), dataset)
             return False
     return False
+
+
+def retrieve_stac_model(model_name, user):
+    repo = ModelsAPIRepo()
+    data, error = repo.retrieve_model(model_name)
+    # print(data, error)
+    if data and data["uid"] != user["uid"]:
+        raise Exception("Model already exists.")
+    if error and error == "Model doesn't exist":
+        # create model
+        data, error = repo.create_stac_model(model_name, user)
+        # print(data, error)
+        if error:
+            raise Exception(error)
+        data["id"] = data["model_id"]
+    return data["id"]
+
+
+@with_auth
+def ingest_stac(stac_catalog, logger=None, user=None):
+    repo, files_repo = ModelsAPIRepo(), FilesAPIRepo()
+    # load catalog
+    logger("Loading STAC catalog...")
+    df = STACDataFrame.from_stac_file(stac_catalog)
+    catalog = df[df["type"] == "Catalog"]
+    assert len(catalog) == 1, "STAC catalog must have exactly one root catalog"
+    dataset_name = catalog.id.iloc[0]
+    # retrieve dataset (create if doesn't exist)
+    model_id = retrieve_stac_model(dataset_name, user)
+    # create new version
+    version = create_new_version(repo, model_id, user)
+    logger("New version created, version: " + str(version))
+    df2 = df.dropna(subset=["assets"])
+    for row in tqdm(df2.iterrows(), total=len(df2)):
+        try:
+            for k, v in row[1]["assets"].items():
+                data, error = files_repo.ingest_file(
+                    v["href"],
+                    model_id,
+                    user,
+                    calculate_checksum(v["href"]),  # is always absolute?
+                    "models",
+                    version,
+                )
+                if error:
+                    raise Exception(error)
+                file_url = (
+                    f"{repo.url}models/{data['model_id']}/download/{data['filename']}"
+                )
+                df.loc[row[0], "assets"][k]["href"] = file_url
+        except Exception as e:
+            logger(f"Error uploading asset {row[0]}: {e}")
+            break
+    # ingest the STAC catalog into geodb
+    logger("Ingesting STAC catalog...")
+    data, error = repo.ingest_stac(json.loads(df.to_json()), model_id, user)
+    if error:
+        # TODO: delete all assets that were uploaded
+        raise Exception(error)
+    logger("Done")
+    return

eotdl/repos/ModelsAPIRepo.py

@@ -1,4 +1,5 @@
 import requests
+import geopandas as gpd
 
 from ..repos import APIRepo
 
@@ -53,3 +54,27 @@ class ModelsAPIRepo(APIRepo):
             headers=self.generate_headers(user),
         )
         return self.format_response(response)
+
+    def create_stac_model(self, name, user):
+        response = requests.post(
+            self.url + "models/stac",
+            json={"name": name},
+            headers=self.generate_headers(user),
+        )
+        return self.format_response(response)
+
+    def ingest_stac(self, stac_json, model_id, user):
+        response = requests.put(
+            self.url + f"models/stac/{model_id}",
+            json={"stac": stac_json},
+            headers=self.generate_headers(user),
+        )
+        return self.format_response(response)
+
+    def download_stac(self, model_id, user):
+        url = self.url + "models/" + model_id + "/download"
+        headers = self.generate_headers(user)
+        response = requests.get(url, headers=headers)
+        if response.status_code != 200:
+            return None, response.json()["detail"]
+        return gpd.GeoDataFrame.from_features(response.json()["features"]), None

{eotdl-2024.5.2.dist-info → eotdl-2024.6.13.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: eotdl
-Version: 2024.5.2
+Version: 2024.6.13
 Summary: Earth Observation Training Data Lab
 License: MIT
 Author: EarthPulse
@@ -12,7 +12,6 @@ Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.12
 Requires-Dist: black (>=23.10.1,<24.0.0)
 Requires-Dist: geomet (>=1.0.0,<2.0.0)
 Requires-Dist: geopandas (>=0.13.2,<0.14.0)

{eotdl-2024.5.2.dist-info → eotdl-2024.6.13.dist-info}/RECORD

@@ -1,4 +1,4 @@
-eotdl/__init__.py,sha256=xrNIo8ojW99vbCIfmq4FSOCpbcqkPd11zXtBDaGUxXA,27
+eotdl/__init__.py,sha256=ThfWXF_wVGcoWrP25NDPXeCmuGEi8P8FKQkVKe_JZZA,27
 eotdl/access/__init__.py,sha256=jbyjD7BRGJURlTNmtcbBBhw3Xk4EiZvkqmEykM-bJ1k,231
 eotdl/access/airbus/__init__.py,sha256=G_kkRS9eFjXbQ-aehmTLXeAxh7zpAxz_rgB7J_w0NRg,107
 eotdl/access/airbus/client.py,sha256=zjfgB_NTsCCIszoQesYkyLJgheKg-eTh28vbleXYxfw,12018
@@ -52,8 +52,8 @@ eotdl/files/__init__.py,sha256=2zfAxgLHmU_jWq_3emnfPXsX-R20gSt-yZX0bPa9h0g,87
 eotdl/files/ingest.py,sha256=dgjZfd-ACCKradDo2B02CPahwEhFtWvnKvTm372K5eo,6185
 eotdl/files/list_files.py,sha256=k4OgdbQ7u6tUEE9nJZGXw5s5HtvG0ZApOVTy0KbfTqs,519
 eotdl/models/__init__.py,sha256=-PvGWG0iSRNBqeFWpxol12dYw-QodXjR81n-JX3x6zI,146
-eotdl/models/download.py,sha256=uXWUslaBkJ222BfctbO0eKNcw6Unx1-Pa9Yv5RHXJIo,4119
-eotdl/models/ingest.py,sha256=KmBFUS1pkUYdegfi0lL1sd__lj9GMoJ7jxvnVXfY8Ts,3381
+eotdl/models/download.py,sha256=4dgxE9ytT8QqiCyx1r19vL5UASfttUN_mCPKhFwMTfs,3410
+eotdl/models/ingest.py,sha256=8xhGlsADi5dZSNbph4WWMk0cs2J_StxNCYRkSjMhUtg,5747
 eotdl/models/metadata.py,sha256=S5bpIB4e2pivDnReszJKC3bYBZcaHu-KMYOc3AwHbQ4,1443
 eotdl/models/retrieve.py,sha256=-Ij7dT4J1p7MW4n13OlPB9OW4tBaBXPwk9dW8IuCZPc,664
 eotdl/models/update.py,sha256=4FWeD95cXvRpefRjw3Foqb30e30otxqWUZ6nQM9cbmM,374
@@ -62,7 +62,7 @@ eotdl/repos/AuthAPIRepo.py,sha256=vYCqFawe3xUm2cx4SqVXCvzl8J_sr9rs_MkipYC0bXE,95
 eotdl/repos/AuthRepo.py,sha256=jpzzhINCcDZHRCyrPDsp49h17IlXp2HvX3BB3f5cnb4,1154
 eotdl/repos/DatasetsAPIRepo.py,sha256=rKqSe-UjJOlD0Kbypu4Gs5kx2mzUD7TY05gv2vgUTv4,2660
 eotdl/repos/FilesAPIRepo.py,sha256=cS6CFTkSYIXcefHEeLt7I69_EyyO-jhoAwQ7UWYlLkc,7171
-eotdl/repos/ModelsAPIRepo.py,sha256=7-0c-3OCTCwKUZ-Z7PWtbRusUxpTU7m2H5jIAcFoJ9M,1677
+eotdl/repos/ModelsAPIRepo.py,sha256=79euf5WsfUxG5KSIGhKT8T7kSl-NtISwxvqHnck-bq0,2616
 eotdl/repos/__init__.py,sha256=WvX5TP49k7yYb5dWWNjv5kzbdluO3dJ4LqjQxRIOUVc,222
 eotdl/shared/__init__.py,sha256=mF7doJC8Z5eTPmB01UQvPivThZac32DRY33T6qshXfg,41
 eotdl/shared/checksum.py,sha256=4IB6N9jRO0chMDNJzpdnFDhC9wcFF9bO5oHq2HodcHw,479
@@ -73,7 +73,7 @@ eotdl/tools/paths.py,sha256=yWhOtVxX4NxrDrrBX2fuye5N1mAqrxXFy_eA7dffd84,1152
 eotdl/tools/stac.py,sha256=ovXdrPm4Sn9AAJmrP88WnxDmq2Ut-xPoscjphxz3Iyo,5763
 eotdl/tools/time_utils.py,sha256=qJ3-rk1I7ne722SLfAP6-59kahQ0vLQqIf9VpOi0Kpg,4691
 eotdl/tools/tools.py,sha256=Tl4_v2ejkQo_zyZek8oofJwoYcdVosdOwW1C0lvWaNM,6354
-eotdl-2024.5.2.dist-info/METADATA,sha256=O_RtJWIg7MvaJ3PeX4ZPIpQHuxUdSSZvdtK6qWb8ciY,4102
-eotdl-2024.5.2.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-eotdl-2024.5.2.dist-info/entry_points.txt,sha256=s6sfxUfRrSX2IP2UbrzTFTvRCtLgw3_OKcHlOKf_5F8,39
-eotdl-2024.5.2.dist-info/RECORD,,
+eotdl-2024.6.13.dist-info/METADATA,sha256=cZRWhvu3uUpHBRQ3D-6hacu4TDy2iU2K48SL7ORBQOw,4052
+eotdl-2024.6.13.dist-info/WHEEL,sha256=Zb28QaM1gQi8f4VCBhsUklF61CTlNYfs9YAZn-TOGFk,88
+eotdl-2024.6.13.dist-info/entry_points.txt,sha256=s6sfxUfRrSX2IP2UbrzTFTvRCtLgw3_OKcHlOKf_5F8,39
+eotdl-2024.6.13.dist-info/RECORD,,

{eotdl-2024.5.2.dist-info → eotdl-2024.6.13.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 1.9.0
+Generator: poetry-core 1.6.1
 Root-Is-Purelib: true
 Tag: py3-none-any