eotdl 2025.4.22.post2__py3-none-any.whl → 2025.5.26__py3-none-any.whl

eotdl/__init__.py

@@ -1 +1 @@
-__version__ = "2025.04.22-2"
+__version__ = "2025.05.26"

eotdl/cli.py

@@ -1,7 +1,7 @@
 import typer
 import os
 
-from .commands import auth, datasets, models, stac
+from .commands import auth, datasets, models, stac, pipelines
 from .repos import APIRepo
 from . import __version__
 
@@ -11,6 +11,7 @@ app.add_typer(auth.app, name="auth")
 app.add_typer(datasets.app, name="datasets")
 app.add_typer(models.app, name="models")
 app.add_typer(stac.app, name="stac")
+app.add_typer(pipelines.app, name="pipelines")
 
 @app.command()
 def version():

eotdl/commands/auth.py

@@ -28,7 +28,9 @@ def login():
 
 
 @app.command()
-def logout():
+def logout(
+    yes: bool = typer.Option(False, "--yes", "-y", help="Skip confirmation")
+):
     """
     Logout from the EOTDL.
 
@@ -37,7 +39,8 @@ def logout():
     user = is_logged()
     if user:
         typer.echo(f"You are logged in as {user['email']}")
-        typer.confirm("Are you sure you want to logout?", abort=True)
+        if not yes:
+            typer.confirm("Are you sure you want to logout?", abort=True)
         logout_url = logout_user()
         typer.echo("You are logged out.")
         typer.echo(

eotdl/commands/datasets.py

@@ -6,6 +6,7 @@ from ..datasets import (
     ingest_dataset,
     stage_dataset,
     deactivate_dataset,
+    retrieve_private_datasets
 )
 
 app = typer.Typer(help="Explore, ingest and download training datasets.")
@@ -72,6 +73,9 @@ def list(
     limit: int = typer.Option(
         None, "--limit", "-l", help="Limit the number of returned results"
     ),
+    private: bool = typer.Option(
+        False, "--private", "-p", help="Show private datasets"
+    ),
 ):
     """
     Retrieve a list with all the datasets in the EOTDL.
@@ -85,7 +89,10 @@ def list(
     $ eotdl datasets list --name YourModel --limit 5
     """
     try:
-        datasets = retrieve_datasets(name, limit)
+        if private:
+            datasets = retrieve_private_datasets()
+        else:
+            datasets = retrieve_datasets(name, limit)
         typer.echo(datasets)
     except Exception as e:
         typer.echo(e)
@@ -96,9 +103,6 @@ def get(
     path: str = typer.Option(
         None, "--path", "-p", help="Download the dataset to a specific output path"
     ),
-    file: str = typer.Option(
-        None, "--file", "-f", help="Download a specific file from the dataset"
-    ),
     version: int = typer.Option(None, "--version", "-v", help="Dataset version"),
     assets: bool = typer.Option(
         False, "--assets", "-a", help="Download STAC assets from the dataset"
@@ -116,7 +120,6 @@ def get(
     Download a dataset from the EOTDL.
     \n\n
     If using --path, it will download the dataset to the specified path. If no path is provided, it will download to ~/.eotdl/datasets.\n
-    If using --file, it will download the specified file. If no file is provided, it will download the entire dataset.\n
     If using --version, it will download the specified version. If no version is provided, it will download the latest version.\n
     If using --assets when the dataset is STAC, it will also download the STAC assets of the dataset. If not provided, it will only download the STAC metadata.\n
     If using --force, it will download the dataset even if the file already exists.\n
@@ -125,7 +128,7 @@ def get(
     Examples\n
     --------\n
     $ eotdl dataset get YourDataset\n
-    $ eotdl dataset get YourDataset --path /path/to/download --file dataset.zip --version 1 --assets True --force True --verbose True
+    $ eotdl dataset get YourDataset --path /path/to/download --version 1 --assets True --force True --verbose True
     """
     try:
         dst_path = stage_dataset(
@@ -143,12 +146,15 @@ def get(
 
 
 @app.command()
-def deactivate(
-    dataset_id: str = typer.Argument(None, help="ID of the dataset to deactivate")
+def delete(
+    dataset: str = typer.Argument(None, help="Name of the dataset to deactivate")
 ):
+    """
+    Delete a dataset from the EOTDL.
+    """
     try:
-        deactivate_dataset(dataset_id)
-        typer.echo(f"Dataset {dataset_id} deactivated")
+        deactivate_dataset(dataset)
+        typer.echo(f"Dataset {dataset} deleted")
     except Exception as e:
         typer.echo(e)
 

eotdl/commands/models.py

@@ -93,9 +93,6 @@ def get(
     path: str = typer.Option(
         None, "--path", "-p", help="Download the model to a specific output path"
     ),
-    file: str = typer.Option(
-        None, "--file", "-f", help="Download a specific file from the model"
-    ),
     version: int = typer.Option(None, "--version", "-v", help="Model version"),
     assets: bool = typer.Option(
         False, "--assets", "-a", help="Download STAC assets from the model"
@@ -113,7 +110,6 @@ def get(
     Download a model from the EOTDL.
     \n\n
     If using --path, it will download the model to the specified path. If no path is provided, it will download to ~/.eotdl/models.\n
-    If using --file, it will download the specified file. If no file is provided, it will download the entire model.\n
     If using --version, it will download the specified version. If no version is provided, it will download the latest version.\n
     If using --assets when the model is STAC, it will also download the STAC assets of the model. If not provided, it will only download the STAC metadata.\n
     If using --force, it will download the model even if the file already exists.\n
@@ -122,7 +118,7 @@ def get(
     Examples\n
     --------\n
     $ eotdl models get YourModel\n
-    $ eotdl models get YourModel --path /path/to/download --file model.zip --version 1 --assets True --force True --verbose True
+    $ eotdl models get YourModel --path /path/to/download --version 1 --assets True --force True --verbose True
     """
     try:
         dst_path = stage_model(
@@ -134,12 +130,15 @@ def get(
 
 
 @app.command()
-def deactivate(
-    model_id: str = typer.Argument(None, help="ID of the model to deactivate")
+def delete(
+    model_name: str = typer.Argument(None, help="Name of the model to delete")
 ):
+    """
+    Delete a model from the EOTDL.
+    """
     try:
-        deactivate_model(model_id)
-        typer.echo(f"Model {model_id} deactivated")
+        deactivate_model(model_name)
+        typer.echo(f"Model {model_name} deleted")
     except Exception as e:
         typer.echo(e)
 

eotdl/commands/pipelines.py

@@ -0,0 +1,132 @@
+import typer
+from pathlib import Path
+
+from ..fe import (
+    ingest_openeo,
+    retrieve_pipelines,
+    stage_pipeline,
+    deactivate_pipeline,
+)
+
+app = typer.Typer(help="Explore, ingest and download Feature Engineering Pipelines.")
+
+@app.command()
+def ingest(
+    path: Path = typer.Option(..., "--path", "-p", help="Path to the pipeline to ingest"),
+    verbose: bool = typer.Option(
+        False,
+        "--verbose",
+        help="Verbose output. This will print the progress of the ingestion",
+    ),
+    foce_metadata_update: bool = typer.Option(
+        False,
+        "--force",
+        "-f",
+        help="Force metadata update even if it already exists. Will overwrite the current metadata in EOTDL",
+    ),
+    sync_metadata: bool = typer.Option(
+        False,
+        "--sync",
+        "-s",
+        help="Sync local metadata with the EOTDL. Will overwrite the local metadata",
+    ),
+):
+    """
+    Ingest a pipeline to the EOTDL.
+
+    This command ingests the pipeline to the EOTDL. The pipeline must be a folder with the pipeline files,
+    and at least a README.md file. All the files in the folder will be uploaded to the EOTDL.
+    \n\n
+    Examples\n
+    --------\n
+    $ eotdl pipelines ingest --path /path/to/folder-with-pipeline --verbose True
+    """
+    try:
+        ingest_openeo(path, verbose, typer.echo, foce_metadata_update, sync_metadata)
+    except Exception as e:
+        typer.echo(e)
+
+@app.command()
+def list(
+    name: str = typer.Option(
+        None, "--name", "-n", help="Filter the returned pipelines by name"
+    ),
+    limit: int = typer.Option(
+        None, "--limit", "-l", help="Limit the number of returned results"
+    ),
+):
+    """
+    Retrieve a list with all the pipelines in the EOTDL.
+
+    If using --name, it will filter the results by name. If no name is provided, it will return all the pipelines.\n
+    If using --limit, it will limit the number of results. If no limit is provided, it will return all the pipelines.
+    \n\n
+    Examples\n
+    --------\n
+    $ eotdl pipelines list\n
+    $ eotdl pipelines list --name YourPipeline --limit 5
+    """
+    try:
+        pipelines = retrieve_pipelines(name, limit)
+        typer.echo(pipelines)
+    except Exception as e:
+        typer.echo(e)
+
+@app.command()
+def get(
+    pipeline: str = typer.Argument(None, help="Name of the pipeline to download"),
+    path: str = typer.Option(
+        None, "--path", "-p", help="Download the pipeline to a specific output path"
+    ),
+    version: int = typer.Option(None, "--version", "-v", help="pipeline version"),
+    assets: bool = typer.Option(
+        False, "--assets", "-a", help="Download STAC assets from the pipeline"
+    ),
+    force: bool = typer.Option(
+        False, "--force", "-f", help="Force download even if file exists"
+    ),
+    verbose: bool = typer.Option(
+        False,
+        "--verbose",
+        help="Verbose output. This will print the progress of the download",
+    ),
+):
+    """
+    Download a pipeline from the EOTDL.
+    \n\n
+    If using --path, it will download the pipeline to the specified path. If no path is provided, it will download to ~/.eotdl/pipelines.\n
+    If using --version, it will download the specified version. If no version is provided, it will download the latest version.\n
+    If using --assets when the pipeline is STAC, it will also download the STAC assets of the pipeline. If not provided, it will only download the STAC metadata.\n
+    If using --force, it will download the pipeline even if the file already exists.\n
+    If using --verbose, it will print the progress of the download.
+    \n\n
+    Examples\n
+    --------\n
+    $ eotdl pipelines get Yourpipeline\n
+    $ eotdl pipelines get Yourpipeline --path /path/to/download  --version 1 --assets True --force True --verbose True
+    """
+    try:
+        dst_path = stage_pipeline(
+            pipeline, version, path, typer.echo, assets, force, verbose
+        )
+        typer.echo(f"Data available at {dst_path}")
+    except Exception as e:
+        typer.echo(e)
+
+
+@app.command()
+def delete(
+    pipeline_name: str = typer.Argument(None, help="Name of the pipeline to delete")
+):
+    """
+    Delete a model from the EOTDL.
+    """
+    try:
+        deactivate_pipeline(pipeline_name)
+        typer.echo(f"Pipeline {pipeline_name} deleted")
+    except Exception as e:
+        typer.echo(e)
+
+
+if __name__ == "__main__":
+    app()

eotdl/datasets/__init__.py

@@ -1,4 +1,4 @@
-from .retrieve import retrieve_datasets, retrieve_dataset, retrieve_dataset_files
+from .retrieve import retrieve_datasets, retrieve_dataset, retrieve_dataset_files, retrieve_private_datasets
 from .ingest import ingest_dataset, ingest_virtual_dataset, ingest_dataset_catalog
 from .stage import stage_dataset, stage_dataset_file
 from .update import deactivate_dataset

eotdl/datasets/ingest.py

@@ -3,16 +3,26 @@ from pathlib import Path
 from ..repos import DatasetsAPIRepo
 from ..files.ingest import prep_ingest_stac, prep_ingest_folder, ingest, ingest_virtual, ingest_catalog
 
-def retrieve_dataset(metadata, user):
+def retrieve_dataset(metadata, user, private):
 	repo = DatasetsAPIRepo()
 	data, error = repo.retrieve_dataset(metadata.name)
 	if data and data["uid"] != user["uid"]:
 		raise Exception("Dataset already exists.")
-	if error and error == "Dataset doesn't exist":
-		# create dataset
-		data, error = repo.create_dataset(metadata.dict(), user)
-		# print(data, error)
-		if error:
+	if error:
+		if error == "Dataset doesn't exist":
+			# create dataset
+			data, error = repo.create_dataset(metadata.dict(), user, private)
+			# print(data, error)
+			if error:
+				raise Exception(error)
+		elif error == "NoAccessToPrivateError":
+			data, error = repo.retrieve_private_dataset(metadata.name, user)
+			if error:
+				if error == "NoAccessToPrivateError":
+					raise Exception("Dataset already exists.")
+				else:	
+					raise Exception(error)
+		else:
 			raise Exception(error)
 	return data
 
@@ -22,6 +32,7 @@ def ingest_dataset(
 	logger=print,
 	force_metadata_update=False,
 	sync_metadata=False,
+	private=False,
 ):
 	path = Path(path)
 	if not path.is_dir():
@@ -30,7 +41,7 @@ def ingest_dataset(
 		prep_ingest_stac(path, logger)
 	else:
 		prep_ingest_folder(path, verbose, logger, force_metadata_update, sync_metadata)
-	return ingest(path, DatasetsAPIRepo(), retrieve_dataset, 'datasets')
+	return ingest(path, DatasetsAPIRepo(), retrieve_dataset, 'datasets', private)
 
 
 def ingest_virtual_dataset( # could work for a list of paths with minimal changes...

eotdl/datasets/retrieve.py

@@ -1,4 +1,5 @@
 from ..repos import DatasetsAPIRepo, FilesAPIRepo
+from ..auth import with_auth
 
 
 def retrieve_datasets(name=None, limit=None):
@@ -10,11 +11,16 @@ def retrieve_datasets(name=None, limit=None):
     return []
 
 
-def retrieve_dataset(name):
+def retrieve_dataset(name, user=None):
     repo = DatasetsAPIRepo()
     data, error = repo.retrieve_dataset(name)
     if error:
-        raise Exception(error)
+        if error == "NoAccessToPrivateError" and user is not None:
+            data, error = repo.retrieve_private_dataset(name, user)
+            if error:
+                raise Exception(error)
+        else:
+            raise Exception(error)
     return data
 
 
@@ -25,3 +31,11 @@ def retrieve_dataset_files(dataset_id, version):
         raise Exception(error)
     return data
 
+@with_auth
+def retrieve_private_datasets(user):
+    api_repo = DatasetsAPIRepo()
+    data, error = api_repo.retrieve_private_datasets(user)
+    if data and not error:
+        datasets = [d["name"] for d in data] if data else []
+        return datasets
+    return []

eotdl/datasets/stage.py

@@ -20,7 +20,7 @@ def stage_dataset(
     user=None,
     file=None,
 ):
-    dataset = retrieve_dataset(dataset_name)
+    dataset = retrieve_dataset(dataset_name, user)
     if version is None:
         version = sorted([v['version_id'] for v in dataset["versions"]])[-1]
     else:

eotdl/datasets/update.py

@@ -1,5 +1,6 @@
 from ..repos import DatasetsAPIRepo
-
+from ..auth import with_auth
+from .retrieve import retrieve_dataset
 
 def update_dataset(dataset_id, metadata, content, user):
     repo = DatasetsAPIRepo()
@@ -16,10 +17,11 @@ def update_dataset(dataset_id, metadata, content, user):
         raise Exception(error)
     return data
 
-
-def deactivate_dataset(dataset_id):
+@with_auth
+def deactivate_dataset(dataset_name, user):
+    dataset = retrieve_dataset(dataset_name)
     repo = DatasetsAPIRepo()
-    data, error = repo.deactivate_dataset(dataset_id)
+    data, error = repo.deactivate_dataset(dataset['id'], user)
     if error:
         raise Exception(error)
     return data

eotdl/fe/__init__.py

@@ -0,0 +1,4 @@
+from .ingest import ingest_openeo
+from .retrieve import retrieve_pipelines, retrieve_pipeline
+from .stage import stage_pipeline
+from .update import deactivate_pipeline

eotdl/fe/ingest.py

@@ -0,0 +1,49 @@
+from pathlib import Path
+
+from ..repos import FEAPIRepo
+from ..files.ingest import prep_ingest_folder, ingest
+
+def retrieve_pipeline(metadata, user):
+	repo = FEAPIRepo()
+	data, error = repo.retrieve_pipeline(metadata.name)
+	if data and data["uid"] != user["uid"]:
+		raise Exception("Pipeline already exists.")
+	if error and error == "Pipeline doesn't exist":
+		# create pipeline
+		data, error = repo.create_pipeline(metadata.dict(), user)
+		# print(data, error)
+		if error:
+			raise Exception(error)
+	return data
+
+def ingest_openeo(
+	path,
+	verbose=False,
+	logger=print,
+	force_metadata_update=False,
+	sync_metadata=False,
+	private=False,
+):
+	path = Path(path)
+	if not path.is_dir():
+		raise Exception("Path must be a folder")
+	# if "catalog.json" in [f.name for f in path.iterdir()]:
+	# 	prep_ingest_stac(path, logger)
+	# else:
+	# 	prep_ingest_folder(path, verbose, logger, force_metadata_update, sync_metadata)
+	prep_ingest_folder(path, verbose, logger, force_metadata_update, sync_metadata)
+	return ingest(path, FEAPIRepo(), retrieve_pipeline, 'pipelines', private)
+
+
+# def ingest_virtual_dataset( # could work for a list of paths with minimal changes...
+# 	path,
+# 	links,
+# 	metadata = None, 
+# 	logger=print,
+# 	user=None,
+# ):
+# 	return ingest_virtual(path, links, DatasetsAPIRepo(), retrieve_dataset, 'datasets', metadata, logger)
+
+# def ingest_dataset_catalog(path, logger=None):
+# 	path = Path(path)
+# 	return ingest_catalog(path, DatasetsAPIRepo(), retrieve_dataset, 'datasets')

eotdl/fe/openeo/__init__.py

@@ -0,0 +1,2 @@
+from .basic_point_extraction import point_extraction
+from .advanced_patch_extraction import patch_extraction

eotdl/fe/openeo/advanced_patch_extraction.py

@@ -0,0 +1,124 @@
+from .dataframe_utils import *
+import openeo
+from openeo.extra.job_management import MultiBackendJobManager, CsvJobDatabase
+from .s3proxy_utils import upload_geoparquet_file
+import geojson
+
+def start_job(row: pd.Series, connection: openeo.Connection, **kwargs) -> openeo.BatchJob:
+
+        temporal_extent = eval(row["temporal_extent"])
+        crs = row['crs']
+
+        # set up load url in order to allow non-latlon feature collections for spatial filtering
+        geometry = geojson.loads(row["geometry"])
+        features = gpd.GeoDataFrame.from_features(geometry).set_crs(crs)
+        url = upload_geoparquet_file(features,connection)
+
+        #run the s1 and s2 udp
+        s1 = connection.datacube_from_process(
+                "s1_weekly_statistics",
+                namespace="https://raw.githubusercontent.com/earthpulse/eotdl/refs/heads/hv_openeoexample/tutorials/notebooks/openeo/s1_weekly_statistics.json",
+                temporal_extent=temporal_extent,
+                )
+        
+        s2 = connection.datacube_from_process(
+                "s2_weekly_statistics",
+                namespace="https://raw.githubusercontent.com/earthpulse/eotdl/refs/heads/hv_openeoexample/tutorials/notebooks/openeo/s2_weekly_statistics.json",
+                temporal_extent=temporal_extent,
+                )
+        
+        #merge both cubes and filter across the feature collection
+        merged = s2.merge_cubes(s1)
+        result = merged.filter_spatial(connection.load_url(url, format="Parquet"))
+        
+        #dedicated job settings to save the individual features within a collection seperately
+        job = result.create_job(
+                out_format="NetCDF",
+                sample_by_feature = True,
+                feature_id_property="id",
+                filename_prefix = "eotdl"
+        )
+
+        return job
+
+def patch_extraction(
+        gdf,
+        start_date,
+        nb_months,
+        pixel_size = 64,
+        resolution = 10,
+        max_points = 5,
+        job_tracker = 'jobs.csv',
+        parallel_jobs=2
+):
+        """
+        # Transform GeoDataFrame for MultiBackendJobManager
+
+        This function processes an input GeoDataFrame and prepares it for use with openEO's **MultiBackendJobManager**. The job manager enables launching and tracking multiple openEO jobs simultaneously, which is essential for large-scale data extractions. 
+
+        ### Example Use Case
+        The function creates patches (e.g., 64x64 pixels) around polygon centers. These patches are suitable for machine learning applications, such as training convolutional neural networks (CNNs).  
+        By combining patches into Sentinel-2 grid collections, the workflow ensures cost efficiency and optimized data extraction.
+
+        ### Workflow
+
+        1. **Process the GeoDataFrame**  
+        - Create patches with a fixed size around the center of polygon geometries.  
+        - Calculate temporal extents for each geometry.  
+
+        2. **Combine Features Using Sentinel-2 Tiling**  
+        - Group buffered geometries into collections based on the Sentinel-2 tiling grid.  
+        - Minimize redundant openEO cost.  
+
+        3. **Generate Job Metadata DataFrame**  
+        - Convert processed data into a DataFrame, ready for the MultiBackendJobManager.
+
+        ### Parameters
+
+        #### Spatial Parameters:
+        - **Buffer Distance:** Buffer size (e.g., 320 meters for a 64x64 patch around polygon centers).  
+        - **Resolution:** Spatial alignment resolution in meters.
+
+        #### Temporal Parameters:
+        - **Start Date:** Start of the temporal extent (e.g., `"2020-01-01"`).  
+        - **Number of Months:** Duration of the temporal extent in months.
+
+        #### Job Splitting Parameters:
+        - **Max Points Per Job:** Maximum number of features per job batch.
+        """
+        job_df = process_and_create_advanced_patch_jobs(
+                gdf, start_date, nb_months, pixel_size, resolution, max_points=max_points
+        )
+        """
+        # Start Job with Standardized UDPs and Feature Collection Filtering
+
+        This function initializes an openEO batch job using standardized **User-Defined Processes (UDPs)** for Sentinel-1 and Sentinel-2 data processing. It employs a spatial filter designed for non-lat/lon feature collections to ensure precise patch sizes in UTM coordinates.
+
+        ### Key Features
+
+        1. **Use of Standardized UDPs**  
+        - **S1 Weekly Statistics:** Computes weekly statistics from Sentinel-1 data.  
+        - **S2 Weekly Statistics:** Computes weekly statistics from Sentinel-2 data.  
+        - UDPs are defined in external JSON files.
+
+        2. **Spatial Filtering with `load_url`**  
+        - Accepts feature collections in **UTM coordinates** to guarantee patches with exact dimensions (e.g., 64x64 meters).  
+        - Features are uploaded as a GeoParquet file to an creodias S3 bucket, enabling spatial filtering directly on the server.
+
+        3. **Cube Merging**  
+        - Merges Sentinel-1 and Sentinel-2 datacubes for combined analysis.
+
+        4. **Job Configuration**  
+        - Saves each feature in the collection as a separate file.  
+        - Outputs results in **NetCDF** format with filenames derived
+        """
+        # Authenticate and add the backend
+        connection = openeo.connect(url="openeo.dataspace.copernicus.eu").authenticate_oidc()
+        # initialize the job manager
+        manager = MultiBackendJobManager()
+        manager.add_backend("cdse", connection=connection, parallel_jobs=parallel_jobs)
+        job_db = CsvJobDatabase(path=job_tracker)
+        if not job_db.exists():
+            df = manager._normalize_df(job_df)
+            job_db.persist(df)
+        manager.run_jobs(start_job=start_job, job_db=job_db)