eotdl 2025.2.10__py3-none-any.whl → 2025.4.2__py3-none-any.whl

eotdl/models/ingest.py

@@ -1,165 +1,42 @@
 from pathlib import Path
-import yaml
-import frontmatter
-from tqdm import tqdm
-import json
-
-from ..auth import with_auth
-from .metadata import Metadata, generate_metadata
-from ..repos import ModelsAPIRepo, FilesAPIRepo
-from ..shared import calculate_checksum
-from ..files import ingest_files, create_new_version
-from .update import update_model
-from ..curation.stac import STACDataFrame
-
-
-def ingest_model(
-    path, verbose=False, logger=print, force_metadata_update=False, sync_metadata=False
-):
-    path = Path(path)
-    if not path.is_dir():
-        raise Exception("Path must be a folder")
-    if "catalog.json" in [f.name for f in path.iterdir()]:
-        return ingest_stac(path / "catalog.json", logger)
-    return ingest_folder(path, verbose, logger, force_metadata_update, sync_metadata)
 
+from ..repos import ModelsAPIRepo
+from ..files.ingest import prep_ingest_stac, prep_ingest_folder, ingest, ingest_virtual
 
 def retrieve_model(metadata, user):
-    repo = ModelsAPIRepo()
-    data, error = repo.retrieve_model(metadata.name)
-    # print(data, error)
-    if data and data["uid"] != user["uid"]:
-        raise Exception("Model already exists.")
-    if error and error == "Model doesn't exist":
-        # create dataset
-        data, error = repo.create_model(metadata.dict(), user)
-        # print(data, error)
-        if error:
-            raise Exception(error)
-        data["id"] = data["model_id"]
-    return data
-
+	repo = ModelsAPIRepo()
+	data, error = repo.retrieve_model(metadata.name)
+	# print(data, error)
+	if data and data["uid"] != user["uid"]:
+		raise Exception("Model already exists.")
+	if error and error == "Model doesn't exist":
+		# create model
+		data, error = repo.create_model(metadata.dict(), user)
+		if error:
+			raise Exception(error)
+	return data
 
-@with_auth
-def ingest_folder(
-    folder,
-    verbose=False,
-    logger=print,
-    force_metadata_update=False,
-    sync_metadata=False,
-    user=None,
+def ingest_model(
+	path,
+	verbose=False,
+	logger=print,
+	force_metadata_update=False,
+	sync_metadata=False,
 ):
-    repo = ModelsAPIRepo()
-    # load metadata
-    try:
-        readme = frontmatter.load(folder.joinpath("README.md"))
-        metadata, content = readme.metadata, readme.content
-        metadata = Metadata(**metadata)
-    except FileNotFoundError:
-        # load metadata (legacy)
-        metadata = (
-            yaml.safe_load(open(folder.joinpath("metadata.yml"), "r").read()) or {}
-        )
-        metadata = Metadata(**metadata)
-        content = None
-    except Exception as e:
-        raise Exception(f"Error loading metadata: {e}")
-    # retrieve model (create if doesn't exist)
-    model = retrieve_model(metadata, user)
-
-    update_metadata = True
-    if "description" in model:
-        # do not do this if the model is new, only if it already exists
-        update_metadata = check_metadata(
-            model, metadata, content, force_metadata_update, sync_metadata, folder
-        )
-    if update_metadata:
-        update_model(model["id"], metadata, content, user)
-    # ingest files
-    return ingest_files(
-        repo, model["id"], folder, verbose, logger, user, endpoint="models"
-    )
-
-
-def check_metadata(
-    dataset, metadata, content, force_metadata_update, sync_metadata, folder
+	path = Path(path)
+	if not path.is_dir():
+		raise Exception("Path must be a folder")
+	if "catalog.json" in [f.name for f in path.iterdir()]:
+		prep_ingest_stac(path, logger)
+	else:
+		prep_ingest_folder(path, verbose, logger, force_metadata_update, sync_metadata)
+	return ingest(path, ModelsAPIRepo(), retrieve_model, 'models')
+
+def ingest_virtual_model( # could work for a list of paths with minimal changes...
+	path,
+	links,
+	metadata = None, 
+	logger=print,
+	user=None,
 ):
-    if (
-        dataset["name"] != metadata.name
-        or dataset["description"] != content
-        or dataset["authors"] != metadata.authors
-        or dataset["source"] != metadata.source
-        or dataset["license"] != metadata.license
-        or dataset["thumbnail"] != metadata.thumbnail
-    ):
-        if not force_metadata_update and not sync_metadata:
-            raise Exception(
-                "The provided metadata is not consistent with the current metadata. Use -f to force metadata update or -s to sync your local metadata."
-            )
-        if force_metadata_update:
-            return True
-        if sync_metadata:
-            generate_metadata(str(folder), dataset)
-            return False
-    return False
-
-
-def retrieve_stac_model(model_name, user):
-    repo = ModelsAPIRepo()
-    data, error = repo.retrieve_model(model_name)
-    # print(data, error)
-    if data and data["uid"] != user["uid"]:
-        raise Exception("Model already exists.")
-    if error and error == "Model doesn't exist":
-        # create model
-        data, error = repo.create_stac_model(model_name, user)
-        # print(data, error)
-        if error:
-            raise Exception(error)
-        data["id"] = data["model_id"]
-    return data["id"]
-
-
-@with_auth
-def ingest_stac(stac_catalog, logger=None, user=None):
-    repo, files_repo = ModelsAPIRepo(), FilesAPIRepo()
-    # load catalog
-    logger("Loading STAC catalog...")
-    df = STACDataFrame.from_stac_file(stac_catalog)
-    catalog = df[df["type"] == "Catalog"]
-    assert len(catalog) == 1, "STAC catalog must have exactly one root catalog"
-    dataset_name = catalog.id.iloc[0]
-    # retrieve dataset (create if doesn't exist)
-    model_id = retrieve_stac_model(dataset_name, user)
-    # create new version
-    version = create_new_version(repo, model_id, user)
-    logger("New version created, version: " + str(version))
-    df2 = df.dropna(subset=["assets"])
-    for row in tqdm(df2.iterrows(), total=len(df2)):
-        try:
-            for k, v in row[1]["assets"].items():
-                data, error = files_repo.ingest_file(
-                    v["href"],
-                    model_id,
-                    user,
-                    calculate_checksum(v["href"]),  # is always absolute?
-                    "models",
-                    version,
-                )
-                if error:
-                    raise Exception(error)
-                file_url = (
-                    f"{repo.url}models/{data['model_id']}/download/{data['filename']}"
-                )
-                df.loc[row[0], "assets"][k]["href"] = file_url
-        except Exception as e:
-            logger(f"Error uploading asset {row[0]}: {e}")
-            break
-    # ingest the STAC catalog into geodb
-    logger("Ingesting STAC catalog...")
-    data, error = repo.ingest_stac(json.loads(df.to_json()), model_id, user)
-    if error:
-        # TODO: delete all assets that were uploaded
-        raise Exception(error)
-    logger("Done")
-    return
+	return ingest_virtual(path, links, ModelsAPIRepo(), retrieve_model, 'models', metadata, logger)

eotdl/models/stage.py

@@ -0,0 +1,63 @@
+import os
+from pathlib import Path
+from tqdm import tqdm
+import geopandas as gpd
+
+from ..auth import with_auth
+from .retrieve import retrieve_model
+from ..repos import FilesAPIRepo
+from ..files.metadata import Metadata
+
+@with_auth
+def stage_model(
+    model_name,
+    version=None,
+    path=None,
+    logger=print,
+    assets=False,
+    force=False,
+    verbose=False,
+    user=None,
+    file=None,
+):
+    model = retrieve_model(model_name)
+    if version is None:
+        version = sorted([v['version_id'] for v in model["versions"]])[-1]
+    else:
+        assert version in [
+            v["version_id"] for v in model["versions"]
+        ], f"Version {version} not found"
+    download_base_path = os.getenv(
+        "EOTDL_DOWNLOAD_PATH", str(Path.home()) + "/.cache/eotdl/models"
+    )
+    if path is None:
+        download_path = download_base_path + "/" + model_name
+    else:
+        download_path = path + "/" + model_name
+    # check if model already exists
+    if os.path.exists(download_path) and not force:
+        os.makedirs(download_path, exist_ok=True)
+        # raise Exception(
+        #     f"model `{model['name']} v{str(version)}` already exists at {download_path}. To force download, use force=True or -f in the CLI."
+        # )
+    
+    # stage metadata
+    repo = FilesAPIRepo()
+    catalog_path = repo.stage_file(model["id"], f"catalog.v{version}.parquet", user, download_path)
+
+    # stage README.md
+    metadata = Metadata(**model['metadata'], name=model['name'])
+    metadata.save_metadata(download_path)
+
+    if assets:
+        gdf = gpd.read_parquet(catalog_path)
+        for _, row in tqdm(gdf.iterrows(), total=len(gdf), desc="Staging assets"):
+            for k, v in row["assets"].items():
+                stage_model_file(v["href"], download_path)
+
+    return download_path
+
+@with_auth
+def stage_model_file(file_url, path, user):
+    repo = FilesAPIRepo()
+    return repo.stage_file_url(file_url, path, user)

eotdl/repos/APIRepo.py

@@ -5,7 +5,7 @@ import requests
 class APIRepo:
     def __init__(self, url=None):
         default_url = "https://api.eotdl.com/"
-        # default_url = "http://localhost:8001/"
+        # default_url = "http://localhost:8000/"
         self.url = url if url else os.getenv("EOTDL_API_URL", default_url)
 
     def format_response(self, response):

eotdl/repos/DatasetsAPIRepo.py

@@ -19,7 +19,15 @@ class DatasetsAPIRepo(APIRepo):
                 url += "&limit=" + str(limit)
         response = requests.get(url)
         return self.format_response(response)
-
+    
+    def retrieve_dataset(self, name):
+        response = requests.get(self.url + "datasets?name=" + name)
+        return self.format_response(response)
+    
+    def get_dataset_by_id(self, dataset_id):
+        response = requests.get(self.url + "datasets/" + dataset_id)
+        return self.format_response(response)
+    
     def create_dataset(self, metadata, user):
         response = requests.post(
             self.url + "datasets",
@@ -28,53 +36,58 @@ class DatasetsAPIRepo(APIRepo):
         )
         return self.format_response(response)
 
-    def retrieve_dataset(self, name):
-        response = requests.get(self.url + "datasets?name=" + name)
-        return self.format_response(response)
-
-    def create_version(self, dataset_id, user):
+    def complete_ingestion(self, dataset_id, version, size, user):
         response = requests.post(
-            self.url + "datasets/version/" + dataset_id,
+            self.url + "datasets/complete/" + dataset_id,
+            json={"version": version, "size": size},
             headers=self.generate_headers(user),
         )
         return self.format_response(response)
 
-    def create_stac_dataset(self, name, user):
-        response = requests.post(
-            self.url + "datasets/stac",
-            json={"name": name},
-            headers=self.generate_headers(user),
-        )
-        return self.format_response(response)
 
-    def ingest_stac(self, stac_json, dataset_id, user):
-        response = requests.put(
-            self.url + f"datasets/stac/{dataset_id}",
-            json={"stac": stac_json},
-            headers=self.generate_headers(user),
-        )
-        return self.format_response(response)
+    # def create_version(self, dataset_id, user):
+    #     response = requests.post(
+    #         self.url + "datasets/version/" + dataset_id,
+    #         headers=self.generate_headers(user),
+    #     )
+    #     return self.format_response(response)
 
-    def download_stac(self, dataset_id, user):
-        url = self.url + "datasets/" + dataset_id + "/download"
-        headers = self.generate_headers(user)
-        response = requests.get(url, headers=headers)
-        if response.status_code != 200:
-            return None, response.json()["detail"]
-        return gpd.GeoDataFrame.from_features(response.json()["features"]), None
+    # def create_stac_dataset(self, name, user):
+    #     response = requests.post(
+    #         self.url + "datasets/stac",
+    #         json={"name": name},
+    #         headers=self.generate_headers(user),
+    #     )
+    #     return self.format_response(response)
 
-    def update_dataset(
-        self, dataset_id, authors, source, license, thumbnail, content, user
-    ):
-        response = requests.put(
-            self.url + f"datasets/{dataset_id}",
-            json={
-                "authors": authors,
-                "source": source,
-                "license": license,
-                "thumbnail": thumbnail,
-                "description": content,
-            },
-            headers=self.generate_headers(user),
-        )
-        return self.format_response(response)
+    # def ingest_stac(self, stac_json, dataset_id, user):
+    #     response = requests.put(
+    #         self.url + f"datasets/stac/{dataset_id}",
+    #         json={"stac": stac_json},
+    #         headers=self.generate_headers(user),
+    #     )
+    #     return self.format_response(response)
+
+    # def download_stac(self, dataset_id, user):
+    #     url = self.url + "datasets/" + dataset_id + "/download"
+    #     headers = self.generate_headers(user)
+    #     response = requests.get(url, headers=headers)
+    #     if response.status_code != 200:
+    #         return None, response.json()["detail"]
+    #     return gpd.GeoDataFrame.from_features(response.json()["features"]), None
+
+    # def update_dataset(
+    #     self, dataset_id, authors, source, license, thumbnail, content, user
+    # ):
+    #     response = requests.put(
+    #         self.url + f"datasets/{dataset_id}",
+    #         json={
+    #             "authors": authors,
+    #             "source": source,
+    #             "license": license,
+    #             "thumbnail": thumbnail,
+    #             "description": content,
+    #         },
+    #         headers=self.generate_headers(user),
+    #     )
+    #     return self.format_response(response)