eotdl 2024.10.7__tar.gz → 2025.3.25__tar.gz

{eotdl-2024.10.7 → eotdl-2025.3.25}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.3
 Name: eotdl
-Version: 2024.10.7
+Version: 2025.3.25
 Summary: Earth Observation Training Data Lab
 License: MIT
 Author: EarthPulse
@@ -13,6 +13,7 @@ Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Requires-Dist: black (>=23.10.1,<24.0.0)
 Requires-Dist: geomet (>=1.0.0,<2.0.0)
 Requires-Dist: geopandas (>=0.13.2,<0.14.0)
@@ -49,10 +50,10 @@ Description-Content-Type: text/markdown
 
 This is the main library and CLI for the **Earth Observation Training Data Lab** (EOTDL), a complete environment that allows you, among other things, to:
 
-- Explore and download Training Datasets (TDS) for Earth Observation (EO) applications.
+- Explore and stage Training Datasets (TDS) for Earth Observation (EO) applications.
 - Create and upload your own TDS by combining and annotating EO data from different sources.
 - Train Machine Learning (ML) models using the hosted TDS in the cloud with multi-GPU machines.
-- Explore and download pre-trianed ML models for EO applications.
+- Explore and stage pre-trianed ML models for EO applications.
 
 In our blog you will find tutorials to learn how leverage the EOTDL to create and use TDS and ML models for your own EO applications.
 

{eotdl-2024.10.7 → eotdl-2025.3.25}/README.md

@@ -15,10 +15,10 @@
 
 This is the main library and CLI for the **Earth Observation Training Data Lab** (EOTDL), a complete environment that allows you, among other things, to:
 
-- Explore and download Training Datasets (TDS) for Earth Observation (EO) applications.
+- Explore and stage Training Datasets (TDS) for Earth Observation (EO) applications.
 - Create and upload your own TDS by combining and annotating EO data from different sources.
 - Train Machine Learning (ML) models using the hosted TDS in the cloud with multi-GPU machines.
-- Explore and download pre-trianed ML models for EO applications.
+- Explore and stage pre-trianed ML models for EO applications.
 
 In our blog you will find tutorials to learn how leverage the EOTDL to create and use TDS and ML models for your own EO applications.
 

eotdl-2025.3.25/eotdl/__init__.py

@@ -0,0 +1 @@
+__version__ = "2025.03.25"

{eotdl-2024.10.7 → eotdl-2025.3.25}/eotdl/access/search.py

@@ -19,8 +19,6 @@ def search_sentinel_imagery(
     evaluate_sentinel_parameters(
         sensor, time_interval, bounding_box, output_needed=False
     )
-
     client = SHClient()
     parameters = SH_PARAMETERS_DICT[sensor]()
-
     return client.search_data(bounding_box, time_interval, parameters)

{eotdl-2024.10.7 → eotdl-2025.3.25}/eotdl/access/sentinelhub/parameters.py

@@ -42,7 +42,7 @@ class SHS2L1CParameters(SHParameters):
         "include": ["id", "properties.datetime", "properties.eo:cloud_cover"],
         "exclude": [],
     }
-
+    FILTER = None
 
 class SHS1Parameters(SHParameters):
     """

{eotdl-2024.10.7 → eotdl-2025.3.25}/eotdl/cli.py

@@ -1,7 +1,7 @@
 import typer
 import os
 
-from .commands import auth, datasets, models
+from .commands import auth, datasets, models, stac
 from .repos import APIRepo
 from . import __version__
 
@@ -10,7 +10,7 @@ app = typer.Typer(help="Welcome to EOTDL. Learn more at https://www.eotdl.com/")
 app.add_typer(auth.app, name="auth")
 app.add_typer(datasets.app, name="datasets")
 app.add_typer(models.app, name="models")
-
+app.add_typer(stac.app, name="stac")
 
 @app.command()
 def version():

{eotdl-2024.10.7 → eotdl-2025.3.25}/eotdl/commands/datasets.py

@@ -4,39 +4,11 @@ from pathlib import Path
 from ..datasets import (
     retrieve_datasets,
     ingest_dataset,
-    download_dataset,
+    stage_dataset,
 )
 
 app = typer.Typer(help="Explore, ingest and download training datasets.")
 
-
-@app.command()
-def list(
-    name: str = typer.Option(
-        None, "--name", "-n", help="Filter the returned datasets by name"
-    ),
-    limit: int = typer.Option(
-        None, "--limit", "-l", help="Limit the number of returned results"
-    ),
-):
-    """
-    Retrieve a list with all the datasets in the EOTDL.
-
-    If using --name, it will filter the results by name. If no name is provided, it will return all the datasets.\n
-    If using --limit, it will limit the number of results. If no limit is provided, it will return all the datasets.
-    \n\n
-    Examples\n
-    --------\n
-    $ eotdl datasets list\n
-    $ eotdl datasets list --name YourModel --limit 5
-    """
-    try:
-        datasets = retrieve_datasets(name, limit)
-        typer.echo(datasets)
-    except Exception as e:
-        typer.echo(e)
-
-
 @app.command()
 def ingest(
     path: Path = typer.Option(
@@ -62,7 +34,7 @@ def ingest(
     ),
 ):
     """
-    Ingest a dataset to the EOTDL.
+    Ingest a dataset to the EOTDL.asdf
 
     This command ingests the dataset to the EOTDL. The dataset must be a folder with the dataset files,
     and at least a README.md file (and a catalog.json file for Q1+). If these files are missing, the ingestion
@@ -90,7 +62,32 @@ def ingest(
         ingest_dataset(path, verbose, typer.echo, foce_metadata_update, sync_metadata)
     except Exception as e:
         typer.echo(e)
+        
+@app.command()
+def list(
+    name: str = typer.Option(
+        None, "--name", "-n", help="Filter the returned datasets by name"
+    ),
+    limit: int = typer.Option(
+        None, "--limit", "-l", help="Limit the number of returned results"
+    ),
+):
+    """
+    Retrieve a list with all the datasets in the EOTDL.
 
+    If using --name, it will filter the results by name. If no name is provided, it will return all the datasets.\n
+    If using --limit, it will limit the number of results. If no limit is provided, it will return all the datasets.
+    \n\n
+    Examples\n
+    --------\n
+    $ eotdl datasets list\n
+    $ eotdl datasets list --name YourModel --limit 5
+    """
+    try:
+        datasets = retrieve_datasets(name, limit)
+        typer.echo(datasets)
+    except Exception as e:
+        typer.echo(e)
 
 @app.command()
 def get(
@@ -130,7 +127,7 @@ def get(
     $ eotdl dataset get YourDataset --path /path/to/download --file dataset.zip --version 1 --assets True --force True --verbose True
     """
     try:
-        dst_path = download_dataset(
+        dst_path = stage_dataset(
             dataset,
             version,
             path,

{eotdl-2024.10.7 → eotdl-2025.3.25}/eotdl/commands/models.py

@@ -4,39 +4,11 @@ from pathlib import Path
 from ..models import (
     retrieve_models,
     ingest_model,
-    download_model,
+    stage_model,
 )
 
 app = typer.Typer(help="Explore, ingest and download ML models.")
 
-
-@app.command()
-def list(
-    name: str = typer.Option(
-        None, "--name", "-n", help="Filter the returned models by name"
-    ),
-    limit: int = typer.Option(
-        None, "--limit", "-l", help="Limit the number of returned results"
-    ),
-):
-    """
-    Retrieve a list with all the models in the EOTDL.
-
-    If using --name, it will filter the results by name. If no name is provided, it will return all the models.\n
-    If using --limit, it will limit the number of results. If no limit is provided, it will return all the models.
-    \n\n
-    Examples\n
-    --------\n
-    $ eotdl models list\n
-    $ eotdl models list --name YourModel --limit 5
-    """
-    try:
-        models = retrieve_models(name, limit)
-        typer.echo(models)
-    except Exception as e:
-        typer.echo(e)
-
-
 @app.command()
 def ingest(
     path: Path = typer.Option(..., "--path", "-p", help="Path to the model to ingest"),
@@ -88,6 +60,31 @@ def ingest(
     except Exception as e:
         typer.echo(e)
 
+@app.command()
+def list(
+    name: str = typer.Option(
+        None, "--name", "-n", help="Filter the returned models by name"
+    ),
+    limit: int = typer.Option(
+        None, "--limit", "-l", help="Limit the number of returned results"
+    ),
+):
+    """
+    Retrieve a list with all the models in the EOTDL.
+
+    If using --name, it will filter the results by name. If no name is provided, it will return all the models.\n
+    If using --limit, it will limit the number of results. If no limit is provided, it will return all the models.
+    \n\n
+    Examples\n
+    --------\n
+    $ eotdl models list\n
+    $ eotdl models list --name YourModel --limit 5
+    """
+    try:
+        models = retrieve_models(name, limit)
+        typer.echo(models)
+    except Exception as e:
+        typer.echo(e)
 
 @app.command()
 def get(
@@ -127,7 +124,7 @@ def get(
     $ eotdl models get YourModel --path /path/to/download --file model.zip --version 1 --assets True --force True --verbose True
     """
     try:
-        dst_path = download_model(
+        dst_path = stage_model(
             model, version, path, typer.echo, assets, force, verbose
         )
         typer.echo(f"Data available at {dst_path}")

eotdl-2025.3.25/eotdl/commands/stac.py

@@ -0,0 +1,57 @@
+import typer
+from typing import Optional
+
+from ..curation.stac.api import api_status, search_stac_columns, retrieve_stac_collections, retrieve_stac_collection, retrieve_stac_items, retrieve_stac_item, search_stac_items
+
+app = typer.Typer(help="EOTDL STAC API")
+
+@app.command()
+def status():
+    try:
+        data = api_status()
+        typer.echo(data)
+    except Exception as e:
+        typer.echo(e)
+        raise typer.Abort()
+
+@app.command()
+def collections():
+    try:
+        data = retrieve_stac_collections()
+        typer.echo(data)
+    except Exception as e:
+        typer.echo(e)
+        raise typer.Abort()
+    
+@app.command()
+def collection(collection_id: str):
+    try:
+        data = retrieve_stac_collection(collection_id)
+        typer.echo(data)
+    except Exception as e:
+        typer.echo(e)
+        raise typer.Abort()
+    
+@app.command()
+def items(collection_id: str):
+    try:
+        data = retrieve_stac_items(collection_id)
+        typer.echo(data)
+    except Exception as e:
+        typer.echo(e)
+
+@app.command()
+def item(collection_id: str, item_id: str):
+    try:
+        data = retrieve_stac_item(collection_id, item_id)
+        typer.echo(data)
+    except Exception as e:
+        typer.echo(e)
+
+@app.command()
+def search(collection_id: str, query: Optional[str] = None):
+    try:
+        data = search_stac_items(collection_id, query)
+        typer.echo(data)
+    except Exception as e:
+        typer.echo(e)

eotdl-2025.3.25/eotdl/curation/stac/__init__.py

@@ -0,0 +1 @@
+from .stac import create_stac_catalog

eotdl-2025.3.25/eotdl/curation/stac/api.py

@@ -0,0 +1,58 @@
+import json
+
+from ...repos import STACAPIRepo
+
+def api_status():
+    repo = STACAPIRepo()
+    data, error = repo.status()
+    if error:
+        raise Exception(error)
+    return data
+
+def retrieve_stac_collections():
+    repo = STACAPIRepo()
+    data, error = repo.collections()
+    if error:
+        raise Exception(error)
+    return data
+
+def retrieve_stac_collection(collection_id):
+    repo = STACAPIRepo()
+    data, error = repo.collection(collection_id)
+    if error:
+        raise Exception(error)
+    return data
+
+def retrieve_stac_items(collection_id):
+    repo = STACAPIRepo()
+    data, error = repo.items(collection_id)
+    if error:
+        raise Exception(error)
+    return data
+
+def retrieve_stac_item(collection_id, item_id):
+    repo = STACAPIRepo()
+    data, error = repo.item(collection_id, item_id)
+    if error:
+        raise Exception(error)
+    return data
+
+def search_stac_items(collection_id, query = None):
+    repo = STACAPIRepo()
+    if query is None:
+        data, error = repo.search_columns(collection_id)
+        if error:
+            raise Exception(error)
+        return data
+    data, error = repo.search(collection_id, str(query))
+    if error:
+        raise Exception(error)
+    return json.loads(data)
+    
+
+def search_stac_columns(collection_id):
+    repo = STACAPIRepo()
+    data, error = repo.search_columns(collection_id)
+    if error:
+        raise Exception(error)
+    return data

eotdl-2025.3.25/eotdl/curation/stac/stac.py

@@ -0,0 +1,33 @@
+import pyarrow.parquet as pq
+import stac_geoparquet
+import json
+from tqdm import tqdm
+import pystac
+from datetime import datetime
+
+def create_stac_catalog(parquet_catalog_path, stac_catalog = None):
+	# parse items and add to collection
+	table = pq.read_table(parquet_catalog_path)
+	items = []
+	for item in tqdm(stac_geoparquet.arrow.stac_table_to_items(table), total=len(table)):
+		item = pystac.Item.from_dict(item)
+		item.validate()
+		# collection.add_item(item)
+		if stac_catalog is not None:
+			stac_catalog.add_item(item)
+		else:
+			items.append(item)
+		# path = "data/stac/" + item["id"] + ".json"
+		# os.makedirs(os.path.dirname(path), exist_ok=True)
+		# with open(path, "w") as f:
+		# 	json.dump(item, f)
+		# # save item
+		# os.makedirs(path, exist_ok=True)
+		# _path = path + '/' + item.id + ".json"
+		# os.makedirs(os.path.dirname(_path), exist_ok=True)
+		# with open(_path, "w") as f:
+		# 	json.dump(item.to_dict(), f)
+	# save catalog
+	if stac_catalog is not None:
+		return stac_catalog
+	return items

{eotdl-2024.10.7 → eotdl-2025.3.25}/eotdl/datasets/__init__.py

@@ -1,3 +1,3 @@
 from .retrieve import retrieve_datasets, retrieve_dataset, retrieve_dataset_files
 from .ingest import ingest_dataset
-from .download import download_dataset, download_file_url
+from .stage import stage_dataset, stage_dataset_file

eotdl-2025.3.25/eotdl/datasets/ingest.py

@@ -0,0 +1,36 @@
+from pathlib import Path
+
+from ..repos import DatasetsAPIRepo
+from ..files.ingest import prep_ingest_stac, prep_ingest_folder, ingest
+
+def retrieve_dataset(metadata, user):
+	repo = DatasetsAPIRepo()
+	data, error = repo.retrieve_dataset(metadata.name)
+	# print(data, error)
+	if data and data["uid"] != user["uid"]:
+		raise Exception("Dataset already exists.")
+	if error and error == "Dataset doesn't exist":
+		# create dataset
+		data, error = repo.create_dataset(metadata.dict(), user)
+		# print(data, error)
+		if error:
+			raise Exception(error)
+	return data
+
+def ingest_dataset(
+	path,
+	verbose=False,
+	logger=print,
+	force_metadata_update=False,
+	sync_metadata=False,
+):
+	path = Path(path)
+	if not path.is_dir():
+		raise Exception("Path must be a folder")
+	if "catalog.json" in [f.name for f in path.iterdir()]:
+		prep_ingest_stac(path, logger)
+	else:
+		prep_ingest_folder(path, verbose, logger, force_metadata_update, sync_metadata)
+	return ingest(path, DatasetsAPIRepo(), retrieve_dataset, 'datasets')
+
+

{eotdl-2024.10.7 → eotdl-2025.3.25}/eotdl/datasets/retrieve.py

@@ -25,12 +25,3 @@ def retrieve_dataset_files(dataset_id, version):
         raise Exception(error)
     return data
 
-
-# def list_datasets(pattern=None):
-#     datasets = retrieve_datasets()
-#     if pattern:
-#         regex = re.compile(rf".*{re.escape(pattern)}.*", re.IGNORECASE)
-#         names = list(datasets.keys())
-#         valid = [name for name in names if regex.search(name)]
-#         return {name: datasets[name] for name in valid}
-#     return datasets

eotdl-2025.3.25/eotdl/datasets/stage.py

@@ -0,0 +1,64 @@
+import os
+from pathlib import Path
+from tqdm import tqdm
+import geopandas as gpd
+
+from ..auth import with_auth
+from .retrieve import retrieve_dataset
+from ..repos import FilesAPIRepo
+
+@with_auth
+def stage_dataset(
+    dataset_name,
+    version=None,
+    path=None,
+    logger=print,
+    assets=False,
+    force=False,
+    verbose=False,
+    user=None,
+    file=None,
+):
+    dataset = retrieve_dataset(dataset_name)
+    if version is None:
+        version = sorted([v['version_id'] for v in dataset["versions"]])[-1]
+    else:
+        assert version in [
+            v["version_id"] for v in dataset["versions"]
+        ], f"Version {version} not found"
+    download_base_path = os.getenv(
+        "EOTDL_DOWNLOAD_PATH", str(Path.home()) + "/.cache/eotdl/datasets"
+    )
+    if path is None:
+        download_path = download_base_path + "/" + dataset_name 
+    else:
+        download_path = path + "/" + dataset_name 
+    # check if dataset already exists
+    if os.path.exists(download_path) and not force:
+        os.makedirs(download_path, exist_ok=True)
+        # raise Exception(
+        #     f"Dataset `{dataset['name']} v{str(version)}` already exists at {download_path}. To force download, use force=True or -f in the CLI."
+        # )
+        raise Exception(
+            f"Dataset `{dataset['name']}` already exists at {download_path}. To force download, use force=True or -f in the CLI."
+        )
+
+    # stage metadata
+    repo = FilesAPIRepo()
+    catalog_path = repo.stage_file(dataset["id"], f"catalog.v{version}.parquet", user, download_path)
+
+    # TODO: stage README.md
+
+    if assets:
+        gdf = gpd.read_parquet(catalog_path)
+        for _, row in tqdm(gdf.iterrows(), total=len(gdf), desc="Staging assets"):
+            for k, v in row["assets"].items():
+                stage_dataset_file(v["href"], download_path)
+
+    return download_path
+
+
+@with_auth
+def stage_dataset_file(file_url, path, user):
+    repo = FilesAPIRepo()
+    return repo.stage_file_url(file_url, path, user)