eotdl 2024.2.15__tar.gz → 2024.3.14.post2__tar.gz

{eotdl-2024.2.15 → eotdl-2024.3.14.post2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: eotdl
-Version: 2024.2.15
+Version: 2024.3.14.post2
 Summary: Earth Observation Training Data Lab
 License: MIT
 Author: EarthPulse
@@ -15,10 +15,13 @@ Classifier: Programming Language :: Python :: 3.11
 Requires-Dist: black (>=23.10.1,<24.0.0)
 Requires-Dist: geomet (>=1.0.0,<2.0.0)
 Requires-Dist: geopandas (>=0.13.2,<0.14.0)
+Requires-Dist: markdown (>=3.5.2,<4.0.0)
+Requires-Dist: markdownify (>=0.11.6,<0.12.0)
 Requires-Dist: mypy (>=1.6.1,<2.0.0)
 Requires-Dist: pydantic (>=1.10.6,<2.0.0)
 Requires-Dist: pyjwt (>=2.6.0,<3.0.0)
 Requires-Dist: pystac[validation] (==1.8.2)
+Requires-Dist: python-frontmatter (>=1.1.0,<2.0.0)
 Requires-Dist: pyyaml (>=6.0.1,<7.0.0)
 Requires-Dist: rasterio (>=1.3.9,<2.0.0)
 Requires-Dist: requests (>=2.28.2,<3.0.0)

eotdl-2024.3.14.post2/eotdl/__init__.py

@@ -0,0 +1 @@
+__version__ = "2024.03.14-2"

{eotdl-2024.2.15 → eotdl-2024.3.14.post2}/eotdl/cli.py

@@ -1,5 +1,8 @@
 import typer
+import os
+
 from .commands import auth, datasets, models
+from .repos import APIRepo
 from . import __version__
 
 app = typer.Typer(help="Welcome to EOTDL. Learn more at https://www.eotdl.com/")
@@ -17,5 +20,15 @@ def version():
     typer.echo(f"EOTDL Version: {__version__}")
 
 
+@app.command()
+def api():
+    """
+    Get EOTDL API URL and info.
+    """
+    repo = APIRepo()
+    typer.echo(f"EOTDL API URL: {repo.url}")
+    typer.echo(repo.get_info())
+
+
 if __name__ == "__main__":
     app()

{eotdl-2024.2.15 → eotdl-2024.3.14.post2}/eotdl/commands/datasets.py

@@ -45,14 +45,27 @@ def ingest(
     verbose: bool = typer.Option(
         False,
         "--verbose",
+        "-v",
         help="Verbose output. This will print the progress of the ingestion",
     ),
+    foce_metadata_update: bool = typer.Option(
+        False,
+        "--force-metadata-update",
+        "-f",
+        help="Force metadata update even if it already exists. Will overwrite the current metadata in EOTDL",
+    ),
+    sync_metadata: bool = typer.Option(
+        False,
+        "--sync-metadata",
+        "-s",
+        help="Sync local metadata with the EOTDL. Will overwrite the local metadata",
+    ),
 ):
     """
     Ingest a dataset to the EOTDL.
 
     This command ingests the dataset to the EOTDL. The dataset must be a folder with the dataset files,
-    and at least a metadata.yml file or a catalog.json file. If there are not these files, the ingestion
+    and at least a README.md file (and a catalog.json file for Q1+). If these files are missing, the ingestion
     will not work. All the files in the folder will be uploaded to the EOTDL.
     \n\n
     The following constraints apply to the dataset name:\n
@@ -60,12 +73,13 @@ def ingest(
     - It must be between 3 and 45 characters long\n
     - It can only contain alphanumeric characters and dashes.\n
     \n
-    The metadata.yml file must contain the following fields:\n
+    The README.md file must contain the following fields in the metadata header:\n
     - name: the name of the dataset\n
     - authors: the author or authors of the dataset\n
     - license: the license of the dataset\n
     - source: the source of the dataset\n
-    \n
+    - thumbnail: an image to use as the thumbnail of the dataset in the website\n
+    The rest of the content in the README.md file will be used as the description of the dataset in the website.
     If using --verbose, it will print the progress of the ingestion.
     \n\n
     Examples\n
@@ -73,7 +87,7 @@ def ingest(
     $ eotdl dataset ingest --path /path/to/folder-with-dataset --verbose True
     """
     try:
-        ingest_dataset(path, verbose, typer.echo)
+        ingest_dataset(path, verbose, typer.echo, foce_metadata_update, sync_metadata)
     except Exception as e:
         typer.echo(e)
 
@@ -108,7 +122,7 @@ def get(
     If using --version, it will download the specified version. If no version is provided, it will download the latest version.\n
     If using --assets when the dataset is STAC, it will also download the STAC assets of the dataset. If not provided, it will only download the STAC metadata.\n
     If using --force, it will download the dataset even if the file already exists.\n
-    If using --verbose, it will print the progress of the download.
+    If using --verbose, it will print the progress of the download.\n
     \n\n
     Examples\n
     --------\n
@@ -117,7 +131,13 @@ def get(
     """
     try:
         dst_path = download_dataset(
-            dataset, version, path, typer.echo, assets, force, verbose
+            dataset,
+            version,
+            path,
+            typer.echo,
+            assets,
+            force,
+            verbose,
         )
         typer.echo(f"Data available at {dst_path}")
     except Exception as e:

{eotdl-2024.2.15 → eotdl-2024.3.14.post2}/eotdl/commands/models.py

@@ -45,6 +45,18 @@ def ingest(
         "--verbose",
         help="Verbose output. This will print the progress of the ingestion",
     ),
+    foce_metadata_update: bool = typer.Option(
+        False,
+        "--force-metadata-update",
+        "-f",
+        help="Force metadata update even if it already exists. Will overwrite the current metadata in EOTDL",
+    ),
+    sync_metadata: bool = typer.Option(
+        False,
+        "--sync-metadata",
+        "-s",
+        help="Sync local metadata with the EOTDL. Will overwrite the local metadata",
+    ),
 ):
     """
     Ingest a model to the EOTDL.
@@ -63,6 +75,7 @@ def ingest(
     - authors: the author or authors of the model\n
     - license: the license of the model\n
     - source: the source of the model\n
+    - thumbnail: an image to use as the thumbnail of the dataset in the website\n
     \n
     If using --verbose, it will print the progress of the ingestion.
     \n\n
@@ -71,7 +84,7 @@ def ingest(
     $ eotdl models ingest --path /path/to/folder-with-model --verbose True
     """
     try:
-        ingest_model(path, verbose, typer.echo)
+        ingest_model(path, verbose, typer.echo, foce_metadata_update, sync_metadata)
     except Exception as e:
         typer.echo(e)
 

{eotdl-2024.2.15 → eotdl-2024.3.14.post2}/eotdl/datasets/download.py

@@ -6,6 +6,7 @@ from ..auth import with_auth
 from .retrieve import retrieve_dataset, retrieve_dataset_files
 from ..repos import FilesAPIRepo, DatasetsAPIRepo
 from ..curation.stac import STACDataFrame
+from .metadata import generate_metadata
 
 
 @with_auth
@@ -61,8 +62,7 @@ def download_dataset(
             )
             # if calculate_checksum(dst_path) != checksum:
             #     logger(f"Checksum for {file} does not match")
-            if verbose:
-                logger("Done")
+
     else:
         # raise NotImplementedError("Downloading a STAC dataset is not implemented")
         if verbose:
@@ -94,6 +94,11 @@ def download_dataset(
         else:
             if verbose:
                 logger("To download assets, set assets=True or -a in the CLI.")
+    if verbose:
+        logger("Generating README.md ...")
+    generate_metadata(download_path, dataset)
+    if verbose:
+        logger("Done")
     return download_path
 
 

{eotdl-2024.2.15 → eotdl-2024.3.14.post2}/eotdl/datasets/ingest.py

@@ -2,6 +2,8 @@ from pathlib import Path
 import yaml
 from tqdm import tqdm
 import json
+import frontmatter
+import markdown
 
 from ..auth import with_auth
 from .metadata import Metadata
@@ -9,15 +11,23 @@ from ..repos import DatasetsAPIRepo, FilesAPIRepo
 from ..files import ingest_files, create_new_version
 from ..curation.stac import STACDataFrame
 from ..shared import calculate_checksum
+from .update import update_dataset
+from .metadata import generate_metadata
 
 
-def ingest_dataset(path, verbose=False, logger=print):
+def ingest_dataset(
+    path,
+    verbose=False,
+    logger=print,
+    force_metadata_update=False,
+    sync_metadata=False,
+):
     path = Path(path)
     if not path.is_dir():
         raise Exception("Path must be a folder")
     if "catalog.json" in [f.name for f in path.iterdir()]:
         return ingest_stac(path / "catalog.json", logger)
-    return ingest_folder(path, verbose, logger)
+    return ingest_folder(path, verbose, logger, force_metadata_update, sync_metadata)
 
 
 def retrieve_dataset(metadata, user):
@@ -33,23 +43,72 @@ def retrieve_dataset(metadata, user):
         if error:
             raise Exception(error)
         data["id"] = data["dataset_id"]
-    return data["id"]
+    return data
 
 
 @with_auth
-def ingest_folder(folder, verbose=False, logger=print, user=None):
+def ingest_folder(
+    folder,
+    verbose=False,
+    logger=print,
+    force_metadata_update=False,
+    sync_metadata=False,
+    user=None,
+):
     repo = DatasetsAPIRepo()
-    # load metadata
-    metadata = yaml.safe_load(open(folder.joinpath("metadata.yml"), "r").read()) or {}
-    metadata = Metadata(**metadata)
+    try:
+        readme = frontmatter.load(folder.joinpath("README.md"))
+        metadata, content = readme.metadata, readme.content
+        metadata = Metadata(**metadata)
+    except FileNotFoundError:
+        # load metadata (legacy)
+        metadata = (
+            yaml.safe_load(open(folder.joinpath("metadata.yml"), "r").read()) or {}
+        )
+        metadata = Metadata(**metadata)
+        content = None
+    except Exception as e:
+        raise Exception("Error loading metadata: " + str(e))
     # retrieve dataset (create if doesn't exist)
-    dataset_id = retrieve_dataset(metadata, user)
-    # ingest files
+    dataset = retrieve_dataset(metadata, user)
+    if content:
+        content = markdown.markdown(content)
+    update_metadata = True
+    if "description" in dataset:
+        # do not do this if the dataset is new, only if it already exists
+        update_metadata = check_metadata(
+            dataset, metadata, content, force_metadata_update, sync_metadata, folder
+        )
+    if update_metadata:
+        update_dataset(dataset["id"], metadata, content, user)
     return ingest_files(
-        repo, dataset_id, folder, verbose, logger, user, endpoint="datasets"
+        repo, dataset["id"], folder, verbose, logger, user, endpoint="datasets"
     )
 
 
+def check_metadata(
+    dataset, metadata, content, force_metadata_update, sync_metadata, folder
+):
+    if (
+        dataset["name"] != metadata.name
+        or dataset["description"] != content
+        or dataset["authors"] != metadata.authors
+        or dataset["source"] != metadata.source
+        or dataset["license"] != metadata.license
+        or dataset["thumbnail"] != metadata.thumbnail
+    ):
+        if not force_metadata_update and not sync_metadata:
+            raise Exception(
+                "The provided metadata is not consistent with the current metadata. Use -f to force metadata update or -s to sync your local metadata."
+            )
+        if force_metadata_update:
+            return True
+        if sync_metadata:
+            generate_metadata(str(folder), dataset)
+            return False
+    return False
+
+
 def retrieve_stac_dataset(dataset_name, user):
     repo = DatasetsAPIRepo()
     data, error = repo.retrieve_dataset(dataset_name)

eotdl-2024.3.14.post2/eotdl/datasets/metadata.py

@@ -0,0 +1,44 @@
+from pydantic import BaseModel, validator
+from typing import List, Optional
+import markdownify
+from pathlib import Path
+
+
+class Metadata(BaseModel):
+    authors: List[str]
+    license: str
+    source: str
+    name: str
+    thumbnail: Optional[str] = ""
+
+    # validate source is a URL
+    @validator("source")
+    def source_is_url(cls, v):
+        if not v.startswith("http") and not v.startswith("https"):
+            raise ValueError("source must be a URL")
+        return v
+
+    # validate thumbnail is a url
+    @validator("thumbnail")
+    def thumbnail_is_url(cls, v):
+        if not v.startswith("http") and not v.startswith("https"):
+            raise ValueError("thumbnail must be a URL")
+        return v
+
+
+def generate_metadata(download_path, dataset):
+    with open(download_path + "/README.md", "w") as f:
+        f.write("---\n")
+        f.write(f"name: {dataset['name']}\n")
+        f.write(f"license: {dataset['license']}\n")
+        f.write(f"source: {dataset['source']}\n")
+        f.write(f"thumbnail: {dataset['thumbnail']}\n")
+        f.write(f"authors:\n")
+        for author in dataset["authors"]:
+            f.write(f"  - {author}\n")
+        f.write("---\n")
+        f.write(markdownify.markdownify(dataset["description"], heading_style="ATX"))
+    # remove metadata.yml if exists
+    if Path(download_path + "/metadata.yml").exists():
+        Path(download_path + "/metadata.yml").unlink()
+    return download_path + "/README.md"

eotdl-2024.3.14.post2/eotdl/datasets/update.py

@@ -0,0 +1,17 @@
+from ..repos import DatasetsAPIRepo
+
+
+def update_dataset(dataset_id, metadata, content, user):
+    repo = DatasetsAPIRepo()
+    data, error = repo.update_dataset(
+        dataset_id,
+        metadata.authors,
+        metadata.source,
+        metadata.license,
+        metadata.thumbnail,
+        content,
+        user,
+    )
+    if error:
+        raise Exception(error)
+    return data

{eotdl-2024.2.15 → eotdl-2024.3.14.post2}/eotdl/files/ingest.py

@@ -13,8 +13,13 @@ from ..shared import calculate_checksum
 def retrieve_files(folder):
     # get all files in directory recursively
     items = [Path(item) for item in glob(str(folder) + "/**/*", recursive=True)]
-    if not any(item.name == "metadata.yml" for item in items):
-        raise Exception("metadata.yml not found in directory")
+    if not any(item.name == "metadata.yml" for item in items) and not any(
+        item.name == "README.md" for item in items
+    ):
+        raise Exception("README.md not found in directory")
+    # remove metadata files
+    items = [item for item in items if item.name != "metadata.yml"]
+    items = [item for item in items if item.name != "README.md"]
     # remove directories
     items = [item for item in items if not item.is_dir()]
     if len(items) == 0:
@@ -129,7 +134,7 @@ def ingest_files(repo, dataset_or_model_id, folder, verbose, logger, user, endpo
                 parts,
                 endpoint,
             )
-            files_repo.complete_upload(user, upload_id, version, endpoint)
+            data, error = files_repo.complete_upload(user, upload_id, version, endpoint)
     # ingest new small files in batches
     if len(upload_files) > 0:
         logger("generating batches...")

{eotdl-2024.2.15 → eotdl-2024.3.14.post2}/eotdl/models/download.py

@@ -6,6 +6,7 @@ from ..auth import with_auth
 from .retrieve import retrieve_model, retrieve_model_files
 from ..shared import calculate_checksum
 from ..repos import FilesAPIRepo
+from .metadata import generate_metadata
 
 
 @with_auth
@@ -75,9 +76,6 @@ def download_model(
             )
             # if calculate_checksum(dst_path) != checksum:
             #     logger(f"Checksum for {file} does not match")
-            if verbose:
-                logger("Done")
-        return "/".join(dst_path.split("/")[:-1])
     else:
         raise NotImplementedError("Downloading a STAC model is not implemented")
     #     logger("Downloading STAC metadata...")
@@ -108,3 +106,9 @@ def download_model(
     #     else:
     #         logger("To download assets, set assets=True or -a in the CLI.")
     #     return Outputs(dst_path=path)
+    if verbose:
+        logger("Generating README.md ...")
+    generate_metadata(download_path, model)
+    if verbose:
+        logger("Done")
+    return download_path

eotdl-2024.3.14.post2/eotdl/models/ingest.py

@@ -0,0 +1,103 @@
+from pathlib import Path
+import yaml
+import frontmatter
+import markdown
+
+from ..auth import with_auth
+from .metadata import Metadata, generate_metadata
+from ..repos import ModelsAPIRepo
+from ..shared import calculate_checksum
+from ..files import ingest_files
+from .update import update_model
+
+
+def ingest_model(
+    path, verbose=False, logger=print, force_metadata_update=False, sync_metadata=False
+):
+    path = Path(path)
+    if not path.is_dir():
+        raise Exception("Path must be a folder")
+    # if "catalog.json" in [f.name for f in path.iterdir()]:
+    #     return ingest_stac(path / "catalog.json", logger)
+    return ingest_folder(path, verbose, logger, force_metadata_update, sync_metadata)
+
+
+def retrieve_model(metadata, user):
+    repo = ModelsAPIRepo()
+    data, error = repo.retrieve_model(metadata.name)
+    # print(data, error)
+    if data and data["uid"] != user["uid"]:
+        raise Exception("Model already exists.")
+    if error and error == "Model doesn't exist":
+        # create dataset
+        data, error = repo.create_model(metadata.dict(), user)
+        # print(data, error)
+        if error:
+            raise Exception(error)
+        data["id"] = data["model_id"]
+    return data
+
+
+@with_auth
+def ingest_folder(
+    folder,
+    verbose=False,
+    logger=print,
+    force_metadata_update=False,
+    sync_metadata=False,
+    user=None,
+):
+    repo = ModelsAPIRepo()
+    # load metadata
+    try:
+        readme = frontmatter.load(folder.joinpath("README.md"))
+        metadata, content = readme.metadata, readme.content
+        metadata = Metadata(**metadata)
+    except FileNotFoundError:
+        # load metadata (legacy)
+        metadata = (
+            yaml.safe_load(open(folder.joinpath("metadata.yml"), "r").read()) or {}
+        )
+        metadata = Metadata(**metadata)
+        content = None
+    except Exception as e:
+        raise Exception("Error loading metadata: " + str(e))
+    # retrieve model (create if doesn't exist)
+    model = retrieve_model(metadata, user)
+    if content:
+        content = markdown.markdown(content)
+    update_metadata = True
+    if "description" in model:
+        # do not do this if the model is new, only if it already exists
+        update_metadata = check_metadata(
+            model, metadata, content, force_metadata_update, sync_metadata, folder
+        )
+    if update_metadata:
+        update_model(model["id"], metadata, content, user)
+    # ingest files
+    return ingest_files(
+        repo, model["id"], folder, verbose, logger, user, endpoint="models"
+    )
+
+
+def check_metadata(
+    dataset, metadata, content, force_metadata_update, sync_metadata, folder
+):
+    if (
+        dataset["name"] != metadata.name
+        or dataset["description"] != content
+        or dataset["authors"] != metadata.authors
+        or dataset["source"] != metadata.source
+        or dataset["license"] != metadata.license
+        or dataset["thumbnail"] != metadata.thumbnail
+    ):
+        if not force_metadata_update and not sync_metadata:
+            raise Exception(
+                "The provided metadata is not consistent with the current metadata. Use -f to force metadata update or -s to sync your local metadata."
+            )
+        if force_metadata_update:
+            return True
+        if sync_metadata:
+            generate_metadata(str(folder), dataset)
+            return False
+    return False

eotdl-2024.3.14.post2/eotdl/models/metadata.py

@@ -0,0 +1,44 @@
+from pydantic import BaseModel, validator
+from typing import List
+import markdownify
+from pathlib import Path
+
+
+class Metadata(BaseModel):
+    authors: List[str]
+    license: str
+    source: str
+    name: str
+    thumbnail: str
+
+    # validate source is a URL
+    @validator("source")
+    def source_is_url(cls, v):
+        if not v.startswith("http") and not v.startswith("https"):
+            raise ValueError("source must be a URL")
+        return v
+
+    # validate thumbnail is a url
+    @validator("thumbnail")
+    def thumbnail_is_url(cls, v):
+        if not v.startswith("http") and not v.startswith("https"):
+            raise ValueError("thumbnail must be a URL")
+        return v
+
+
+def generate_metadata(download_path, model):
+    with open(download_path + "/README.md", "w") as f:
+        f.write("---\n")
+        f.write(f"name: {model['name']}\n")
+        f.write(f"license: {model['license']}\n")
+        f.write(f"source: {model['source']}\n")
+        f.write(f"thumbnail: {model['thumbnail']}\n")
+        f.write(f"authors:\n")
+        for author in model["authors"]:
+            f.write(f"  - {author}\n")
+        f.write("---\n")
+        f.write(markdownify.markdownify(model["description"], heading_style="ATX"))
+    # remove metadata.yml if exists
+    if Path(download_path + "/metadata.yml").exists():
+        Path(download_path + "/metadata.yml").unlink()
+    return download_path + "/README.md"

eotdl-2024.3.14.post2/eotdl/models/update.py

@@ -0,0 +1,17 @@
+from ..repos import ModelsAPIRepo
+
+
+def update_model(model_id, metadata, content, user):
+    repo = ModelsAPIRepo()
+    data, error = repo.update_model(
+        model_id,
+        metadata.authors,
+        metadata.source,
+        metadata.license,
+        metadata.thumbnail,
+        content,
+        user,
+    )
+    if error:
+        raise Exception(error)
+    return data

{eotdl-2024.2.15 → eotdl-2024.3.14.post2}/eotdl/repos/APIRepo.py

@@ -1,4 +1,5 @@
 import os
+import requests
 
 
 class APIRepo:
@@ -18,3 +19,7 @@ class APIRepo:
         if "id_token" in data:
             return {"Authorization": "Bearer " + data["id_token"]}
         raise Exception("Invalid headers")
+
+    def get_info(self):
+        response = requests.get(self.url)
+        return self.format_response(response)

{eotdl-2024.2.15 → eotdl-2024.3.14.post2}/eotdl/repos/DatasetsAPIRepo.py

@@ -62,3 +62,19 @@ class DatasetsAPIRepo(APIRepo):
         if response.status_code != 200:
             return None, response.json()["detail"]
         return gpd.GeoDataFrame.from_features(response.json()["features"]), None
+
+    def update_dataset(
+        self, dataset_id, authors, source, license, thumbnail, content, user
+    ):
+        response = requests.put(
+            self.url + f"datasets/{dataset_id}",
+            json={
+                "authors": authors,
+                "source": source,
+                "license": license,
+                "thumbnail": thumbnail,
+                "description": content,
+            },
+            headers=self.generate_headers(user),
+        )
+        return self.format_response(response)

{eotdl-2024.2.15 → eotdl-2024.3.14.post2}/eotdl/repos/ModelsAPIRepo.py

@@ -37,3 +37,19 @@ class ModelsAPIRepo(APIRepo):
             headers=self.generate_headers(user),
         )
         return self.format_response(response)
+
+    def update_model(
+        self, model_id, authors, source, license, thumbnail, content, user
+    ):
+        response = requests.put(
+            self.url + f"models/{model_id}",
+            json={
+                "authors": authors,
+                "source": source,
+                "license": license,
+                "thumbnail": thumbnail,
+                "description": content,
+            },
+            headers=self.generate_headers(user),
+        )
+        return self.format_response(response)

{eotdl-2024.2.15 → eotdl-2024.3.14.post2}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "eotdl"
-version = "2024.02.15"
+version = "2024.03.14-2"
 description = "Earth Observation Training Data Lab"
 authors = ["EarthPulse <it@earthpulse.es>"]
 license = "MIT"
@@ -25,6 +25,9 @@ sentinelhub = "^3.9.1"
 pyyaml = "^6.0.1"
 black = "^23.10.1"
 mypy = "^1.6.1"
+python-frontmatter = "^1.1.0"
+markdown = "^3.5.2"
+markdownify = "^0.11.6"
 
 [tool.poetry.group.dev.dependencies]
 pytest = "^7.2.2"

eotdl-2024.2.15/eotdl/__init__.py

@@ -1 +0,0 @@
-__version__ = "2024.02.15"

eotdl-2024.2.15/eotdl/datasets/metadata.py

@@ -1,16 +0,0 @@
-from pydantic import BaseModel, validator
-from typing import List
-
-
-class Metadata(BaseModel):
-    authors: List[str]
-    license: str
-    source: str
-    name: str
-
-    # validate source is a URL
-    @validator("source")
-    def source_is_url(cls, v):
-        if not v.startswith("http") and not v.startswith("https"):
-            raise ValueError("source must be a URL")
-        return v

eotdl-2024.2.15/eotdl/models/ingest.py

@@ -1,47 +0,0 @@
-from pathlib import Path
-import yaml
-
-from ..auth import with_auth
-from .metadata import Metadata
-from ..repos import ModelsAPIRepo
-from ..shared import calculate_checksum
-from ..files import ingest_files
-
-
-def ingest_model(path, verbose=False, logger=print):
-    path = Path(path)
-    if not path.is_dir():
-        raise Exception("Path must be a folder")
-    # if "catalog.json" in [f.name for f in path.iterdir()]:
-    #     return ingest_stac(path / "catalog.json", logger)
-    return ingest_folder(path, verbose, logger)
-
-
-def retrieve_model(metadata, user):
-    repo = ModelsAPIRepo()
-    data, error = repo.retrieve_model(metadata.name)
-    # print(data, error)
-    if data and data["uid"] != user["uid"]:
-        raise Exception("Model already exists.")
-    if error and error == "Model doesn't exist":
-        # create dataset
-        data, error = repo.create_model(metadata.dict(), user)
-        # print(data, error)
-        if error:
-            raise Exception(error)
-        data["id"] = data["model_id"]
-    return data["id"]
-
-
-@with_auth
-def ingest_folder(folder, verbose=False, logger=print, user=None):
-    repo = ModelsAPIRepo()
-    # load metadata
-    metadata = yaml.safe_load(open(folder.joinpath("metadata.yml"), "r").read()) or {}
-    metadata = Metadata(**metadata)
-    # retrieve model (create if doesn't exist)
-    model_id = retrieve_model(metadata, user)
-    # ingest files
-    return ingest_files(
-        repo, model_id, folder, verbose, logger, user, endpoint="models"
-    )

eotdl-2024.2.15/eotdl/models/metadata.py

@@ -1,16 +0,0 @@
-from pydantic import BaseModel, validator
-from typing import List
-
-
-class Metadata(BaseModel):
-    authors: List[str]
-    license: str
-    source: str
-    name: str
-
-    # validate source is a URL
-    @validator("source")
-    def source_is_url(cls, v):
-        if not v.startswith("http") and not v.startswith("https"):
-            raise ValueError("source must be a URL")
-        return v