eotdl 2025.4.22.post3__py3-none-any.whl → 2025.5.26.post2__py3-none-any.whl

eotdl/fe/openeo/dataframe_utils.py

@@ -0,0 +1,230 @@
+
+from .temporal_utils import compute_temporal_extent
+from .spatial_utils import  buffer_geometry
+
+import os
+from typing import List
+import geopandas as gpd
+
+import pandas as pd
+from typing import List, Dict, Any
+
+try:
+    import openeo_gfmap
+except ImportError:
+    print("openeo_gfmap is not installed, please install it with 'pip install openeo_gfmap'")
+
+from openeo_gfmap.manager.job_splitters import split_job_s2grid, append_h3_index
+
+def create_utm_patch(
+    row: Any,
+    parent_crs: str,
+    start_date: str,
+    nb_months: int,
+    pixel_size: int,
+    resolution: int
+) -> Dict[str, Any]:
+    """
+    Process a single row from a GeoDataFrame to extract metadata and buffer geometry.
+    
+    Parameters:
+        row: A single row of a GeoDataFrame.
+        start_date: Start date for the temporal extent (ISO format string).
+        nb_months: Number of months for the temporal extent.
+        buffer_distance: Buffer distance in meters for the geometry.
+    
+    Returns:
+        A dictionary with metadata and processed geometry.
+    """
+    # Extract the polygon geometry from the row
+    # Create a single-row GeoDataFrame
+    gdf = gpd.GeoDataFrame(
+        [row], 
+        crs=parent_crs, 
+        geometry="geometry"
+    )
+    
+    buffer = int(pixel_size*resolution/2)
+    # Apply the buffer_geometry function
+    buffered_gdf = buffer_geometry(gdf, buffer=buffer, resolution = resolution)
+
+    # Compute the temporal extent
+    temporal_extent = compute_temporal_extent(start_date, nb_months)
+
+    # Return the processed data
+    return {
+        "fid": row.get("fid"),  # Include any relevant identifier
+        "geometry": buffered_gdf.iloc[0].geometry,
+        "crs": buffered_gdf.crs.to_string(),
+        "temporal_extent": temporal_extent,
+    }
+
+def process_patch_geodataframe(
+    geodataframe: gpd.GeoDataFrame,
+    start_date: str,
+    nb_months: int,
+    buffer_distance: int,
+    resolution: int
+) -> gpd.GeoDataFrame:
+    """
+    Process a GeoDataFrame to generate buffered geometries and metadata for each row.
+    
+    Parameters:
+        geodataframe: The input GeoDataFrame with geometries to process.
+        start_date: Start date for the temporal extent (ISO format string).
+        nb_months: Number of months for the temporal extent.
+        buffer_distance: Buffer distance in meters for the geometry.
+        resolution: Spatial resolution for geometry rounding.
+    
+    Returns:
+        A processed GeoDataFrame with buffered geometries and additional metadata.
+    """
+    results = []  # List to store processed rows
+    
+    for _, row in geodataframe.iterrows():
+        # Process each row and collect the result
+        result = create_utm_patch(row, geodataframe.crs, start_date, nb_months, buffer_distance, resolution)
+        results.append(result)
+    
+    # Convert the list of results into a GeoDataFrame
+    processed_gdf = gpd.GeoDataFrame(
+        results, 
+        geometry="geometry", 
+        crs=result['crs']
+    )
+    
+    return processed_gdf
+
+def process_geodataframe(
+    geodataframe: gpd.GeoDataFrame,
+    start_date: str,
+    nb_months: int,
+    extra_cols: List[str] = [],
+
+) -> gpd.GeoDataFrame:
+    """
+    Process a GeoDataFrame to generate buffered geometries and metadata for each row.
+    
+    Parameters:
+        geodataframe: The input GeoDataFrame with geometries to process.
+        start_date: Start date for the temporal extent (ISO format string).
+        nb_months: Number of months for the temporal extent.
+        buffer_distance: Buffer distance in meters for the geometry.
+        resolution: Spatial resolution for geometry rounding.
+    
+    Returns:
+        A processed GeoDataFrame with buffered geometries and additional metadata.
+    """
+    results = []  # List to store processed rows
+    geodataframe = geodataframe.to_crs(epsg=4326)
+    
+    for _, row in geodataframe.iterrows():
+    
+        # Compute the temporal extent
+        temporal_extent = compute_temporal_extent(start_date, nb_months)
+
+        # Return the processed data
+        result =  {
+            # "fid": row.get("fid"),  # Include any relevant identifier
+            "geometry": row.geometry,
+            "crs": geodataframe.crs.to_string(),
+            "temporal_extent": temporal_extent,
+            **{col: row[col] for col in extra_cols}
+        }  
+        
+        results.append(result)
+    
+    # Convert the list of results into a GeoDataFrame
+    processed_gdf = gpd.GeoDataFrame(
+        results, 
+        geometry="geometry", 
+        crs=result['crs']
+    )
+    
+    return processed_gdf
+
+def split_geodataframe_by_s2_grid(base_gdf: gpd.GeoDataFrame, max_points:int, grid_resolution:int = 3) -> List[gpd.GeoDataFrame]:
+    """Append H3 index and split into smaller job dataframes."""
+    original_crs = base_gdf.crs
+    # Append H3 index, which will change the CRS temporarily
+    base_gdf = append_h3_index(base_gdf, grid_resolution=grid_resolution)
+    # Transform back to the original CRS
+    h3_gdf = base_gdf.to_crs(original_crs)
+    split_gdf = split_job_s2grid(h3_gdf, max_points=max_points)
+    return split_gdf
+
+#TODO evaluate need
+def generate_featurecollection_dataframe(split_jobs: List[gpd.GeoDataFrame]) -> pd.DataFrame:
+    """Create a DataFrame from split jobs with essential information for each job, including feature count."""
+    job_data = []
+    for job in split_jobs:
+
+        # Ensure the temporal_extent field exists and handle missing data
+        temporal_extent = job.temporal_extent.iloc[0] if 'temporal_extent' in job.columns and job.temporal_extent.iloc[0] else None
+        
+        # Handle missing S2 and H3 information gracefully
+        s2_tile = job.tile.iloc[0] if 'tile' in job.columns and job.tile.iloc[0] else None
+        h3index = job.h3index.iloc[0] if 'h3index' in job.columns and job.h3index.iloc[0] else None
+        
+        # Extract CRS as string
+        crs = job.crs.to_string() if job.crs else None
+        
+        # Count the number of features (rows) in the GeoDataFrame
+        feature_count = len(job)
+        
+        # Serialize the entire GeoDataFrame to GeoJSON (including geometry and attributes)
+        job_json = job.to_json()
+
+        # Append all information, including feature count
+        job_data.append({
+            'temporal_extent': temporal_extent,
+            'geometry': job_json,  # The entire GeoDataFrame serialized to GeoJSON
+            's2_tile': s2_tile,
+            'h3index': h3index,
+            'crs': crs,
+            'feature_count': feature_count  # Include the feature count for the job
+        })
+
+    # Return the DataFrame with all job metadata
+    return pd.DataFrame(job_data)
+
+
+def process_and_create_advanced_patch_jobs(
+    gdf: gpd.GeoDataFrame, 
+    start_date: str, 
+    nb_months: int, 
+    pixel_size: int, 
+    resolution: int, 
+    max_points: int
+) -> pd.DataFrame:
+    """
+    A wrapper function to process geospatial data and generate a job metadata DataFrame.
+    
+    Args:
+        gdf (GeoDataFrame): The input GeoDataFrame with geometries.
+        start_date (str): The start date for the temporal extent (ISO format).
+        nb_months (int): Number of months for the temporal extent.
+        buffer_distance (int): Buffer distance in meters for geometry.
+        resolution (int): Spatial resolution for geometry rounding.
+        max_points (int): Maximum number of points per job for splitting the grid.
+    
+    Returns:
+        pd.DataFrame: A DataFrame containing the job metadata.
+    """
+    # Step 1: Process GeoDataFrame in which we create patches and temporal info
+    processed_gdf = process_patch_geodataframe(
+        gdf, start_date, nb_months, pixel_size, resolution
+    )
+
+    # Step 2: Split processed GeoDataFrame into smaller jobs by Sentinel-2 grid
+    split_jobs = split_geodataframe_by_s2_grid(processed_gdf, max_points)
+
+    # Step 3: Generate the job metadata DataFrame
+    job_df = generate_featurecollection_dataframe(split_jobs)
+
+    return job_df
+
+
+
+
+

eotdl/fe/openeo/s3proxy_utils.py

@@ -0,0 +1,180 @@
+from __future__ import (
+    annotations,  # Required because of type annotations for return results
+)
+
+import boto3
+from boto3.s3.transfer import TransferConfig
+from dataclasses import dataclass
+import os
+import hashlib
+import datetime
+import geopandas as gpd
+from openeo.rest.connection import Connection
+from tempfile import NamedTemporaryFile
+
+def upload_geoparquet_artifactory(gdf: gpd.GeoDataFrame, row_id: int) -> str:
+    # Save the dataframe as geoparquet to upload it to artifactory
+    temporary_file = NamedTemporaryFile()
+    gdf.to_parquet(temporary_file.name)
+
+
+
+@dataclass(frozen=True)
+class S3URI:
+    bucket: str
+    key: str
+
+    @classmethod
+    def from_str(cls, uri: str) -> S3URI:
+        s3_prefix = "s3://"
+        if uri.startswith(s3_prefix):
+            without_prefix = uri[len(s3_prefix) :]
+            without_prefix_parts = without_prefix.split("/")
+            bucket = without_prefix_parts[0]
+            if len(without_prefix_parts) == 1:
+                return S3URI(bucket, "")
+            else:
+                return S3URI(bucket, "/".join(without_prefix_parts[1:]))
+        else:
+            raise ValueError(
+                "Input {uri} is not a valid S3 URI should be of form s3://<bucket>/<key>"
+            )
+
+
+@dataclass(frozen=True)
+class AWSSTSCredentials:
+    AWS_ACCESS_KEY_ID: str
+    AWS_SECRET_ACCESS_KEY: str
+    AWS_SESSION_TOKEN: str
+    subject_from_web_identity_token: str
+    STS_ENDPOINT = "https://sts.prod.warsaw.openeo.dataspace.copernicus.eu"
+
+    @classmethod
+    def _from_assume_role_response(cls, resp: dict) -> AWSSTSCredentials:
+        d = resp["Credentials"]
+
+        return AWSSTSCredentials(
+            AWS_ACCESS_KEY_ID=d["AccessKeyId"],
+            AWS_SECRET_ACCESS_KEY=d["SecretAccessKey"],
+            AWS_SESSION_TOKEN=d["SessionToken"],
+            subject_from_web_identity_token=resp["SubjectFromWebIdentityToken"],
+        )
+
+    def set_as_environment_variables(self) -> None:
+        """If temporary credentials are to be used elsewhere in the notebook"""
+        os.environ["AWS_ACCESS_KEY_ID"] = self.AWS_ACCESS_KEY_ID
+        os.environ["AWS_SECRET_ACCESS_KEY"] = self.AWS_SECRET_ACCESS_KEY
+        os.environ["AWS_SESSION_TOKEN"] = self.AWS_SESSION_TOKEN
+
+    def as_kwargs(self) -> dict:
+        return {
+            "aws_access_key_id": self.AWS_ACCESS_KEY_ID,
+            "aws_secret_access_key": self.AWS_SECRET_ACCESS_KEY,
+            "aws_session_token": self.AWS_SESSION_TOKEN,
+        }
+
+    @classmethod
+    def from_openeo_connection(cls, conn: Connection) -> AWSSTSCredentials:
+        """
+        Takes an OpenEO connection object and returns temporary credentials to interact with S3
+        """
+        auth_token = conn.auth.bearer.split("/")
+        os.environ["AWS_ENDPOINT_URL_STS"] = cls.STS_ENDPOINT
+        sts = boto3.client("sts")
+        return AWSSTSCredentials._from_assume_role_response(
+            sts.assume_role_with_web_identity(
+                RoleArn="arn:aws:iam::000000000000:role/S3Access",
+                RoleSessionName=auth_token[1],
+                WebIdentityToken=auth_token[2],
+                DurationSeconds=43200,
+            )
+        )
+
+    def get_user_hash(self) -> str:
+        hash_object = hashlib.sha1(self.subject_from_web_identity_token.encode())
+        return hash_object.hexdigest()
+
+
+class OpenEOArtifactHelper:
+    BUCKET_NAME = "OpenEO-artifacts"
+    S3_ENDPOINT = "https://s3.prod.warsaw.openeo.dataspace.copernicus.eu"
+    # From what size will we switch to multi-part-upload
+    MULTIPART_THRESHOLD_IN_MB = 50
+
+    def __init__(self, creds: AWSSTSCredentials):
+        self._creds = creds
+        self.session = boto3.Session(**creds.as_kwargs())
+
+    @classmethod
+    def from_openeo_connection(cls, conn: Connection) -> OpenEOArtifactHelper:
+        creds = AWSSTSCredentials.from_openeo_connection(conn)
+        return OpenEOArtifactHelper(creds)
+
+    def get_s3_client(self):
+        return self.session.client("s3", endpoint_url=self.S3_ENDPOINT)
+
+    def set_env(self):
+        os.environ["AWS_ENDPOINT_URL_S3"] = self.S3_ENDPOINT
+
+    def user_prefix(self) -> str:
+        """Each user has its own prefix retrieve it"""
+        return self._creds.get_user_hash()
+
+    def get_upload_prefix(self):
+        return (
+            f"{self.user_prefix()}/{datetime.datetime.utcnow().strftime('%Y/%m/%d')}/"
+        )
+
+    def get_upload_key(self, object_name: str) -> str:
+        return f"{self.get_upload_prefix()}{object_name}"
+
+    def upload_bytes(self, object_name: str, blob: bytes) -> str:
+        """Upload a bunch of bytes into an object and return an S3 URI to it"""
+        bucket = self.BUCKET_NAME
+        key = self.get_upload_key(object_name)
+        self.get_s3_client().put_object(Body=blob, Bucket=bucket, Key=key)
+        return f"s3://{bucket}/{key}"
+
+    def upload_string(self, object_name: str, s: str, encoding: str = "utf-8") -> str:
+        """Upload a string into an object"""
+        return self.upload_bytes(object_name, s.encode(encoding))
+
+    def upload_file(self, object_name: str, src_file_path: str) -> str:
+        MB = 1024**2
+        config = TransferConfig(multipart_threshold=self.MULTIPART_THRESHOLD_IN_MB * MB)
+
+        bucket = self.BUCKET_NAME
+        key = self.get_upload_key(object_name)
+
+        self.get_s3_client().upload_file(src_file_path, bucket, key, Config=config)
+        return f"s3://{bucket}/{key}"
+
+    def get_presigned_url(
+        self, s3_uri: str, expires_in_seconds: int = 3600 * 24 * 6
+    ) -> str:
+        typed_s3_uri = S3URI.from_str(s3_uri)
+        return self.get_s3_client().generate_presigned_url(
+            "get_object",
+            Params={"Bucket": typed_s3_uri.bucket, "Key": typed_s3_uri.key},
+            ExpiresIn=expires_in_seconds,
+        )
+    
+
+
+def upload_geoparquet_file(gdf: gpd.GeoDataFrame, conn: Connection) -> str:
+    geo_parquet_path = "bounding_box_geometry.parquet"
+    gdf.to_parquet(geo_parquet_path)
+
+    # Create an instance of the custom S3 uploader
+    uploader = OpenEOArtifactHelper.from_openeo_connection(conn)
+
+    # Upload the GeoParquet file to S3
+    s3_uri = uploader.upload_file(geo_parquet_path, geo_parquet_path)
+
+    # Get the presigned URL for accessing the uploaded file
+    presigned_url = uploader.get_presigned_url(s3_uri)
+
+    # Optionally clean up the local file
+    os.remove(geo_parquet_path)
+
+    return presigned_url

eotdl/fe/openeo/spatial_utils.py

@@ -0,0 +1,28 @@
+import geopandas as gpd
+from shapely.geometry import Point
+
+def buffer_geometry(gdf: gpd.GeoDataFrame, buffer: int, resolution: int) -> gpd.GeoDataFrame:
+    """
+    Buffer the geometries in a GeoDataFrame and return the modified GeoDataFrame.
+    
+    Parameters:
+        gdf: Input GeoDataFrame with geometries to buffer.
+        buffer: Buffer distance in meters.
+    
+    Returns:
+        A GeoDataFrame with buffered geometries.
+    """
+    # Ensure the GeoDataFrame has a valid CRS
+    if gdf.crs is None:
+        raise ValueError("Input GeoDataFrame must have a defined CRS.")
+
+    # Estimate the UTM CRS based on the data's centroid
+    utm = gdf.estimate_utm_crs()
+    gdf_utm = gdf.to_crs(utm)
+    
+    # Round the centroids to the nearest 20m grid and apply buffering
+    gdf_utm['geometry'] = gdf_utm.centroid.apply(
+        lambda point: Point(round(point.x / resolution) * resolution, round(point.y / resolution) * resolution)
+    ).buffer(distance=buffer, cap_style=3)
+    
+    return gdf_utm

eotdl/fe/openeo/temporal_utils.py

@@ -0,0 +1,16 @@
+import pandas as pd
+from typing import List
+
+def compute_temporal_extent(start_date: str, nb_months: int) -> List[str]:
+    """Compute temporal extent based on a start date and duration in months.
+
+    Args:
+        start_date (str): Start date in 'YYYY-MM-DD' format.
+        nb_months (int): Number of months from the start date.
+
+    Returns:
+        List[str]: Temporal extent as [start_date, end_date].
+    """
+    start = pd.to_datetime(start_date)
+    end = start + pd.DateOffset(months=nb_months)
+    return [start.strftime('%Y-%m-%d'), end.strftime('%Y-%m-%d')]

eotdl/fe/retrieve.py

@@ -0,0 +1,18 @@
+from ..repos import FEAPIRepo
+
+
+def retrieve_pipelines(name=None, limit=None):
+    api_repo = FEAPIRepo()
+    data, error = api_repo.retrieve_pipelines(name, limit)
+    if data and not error:
+        models = [d["name"] for d in data] if data else []
+        return models
+    return []
+
+
+def retrieve_pipeline(name):
+    repo = FEAPIRepo()
+    data, error = repo.retrieve_pipeline(name)
+    if error:
+        raise Exception(error)
+    return data

eotdl/fe/stage.py

@@ -0,0 +1,63 @@
+import os
+from pathlib import Path
+from tqdm import tqdm
+import geopandas as gpd
+
+from ..auth import with_auth
+from .retrieve import retrieve_pipeline
+from ..repos import FilesAPIRepo
+from ..files.metadata import Metadata
+
+@with_auth
+def stage_pipeline(
+    pipeline_name,
+    version=None,
+    path=None,
+    logger=print,
+    assets=False,
+    force=False,
+    verbose=False,
+    user=None,
+    file=None,
+):
+    pipeline = retrieve_pipeline(pipeline_name)
+    if version is None:
+        version = sorted([v['version_id'] for v in pipeline["versions"]])[-1]
+    else:
+        assert version in [
+            v["version_id"] for v in pipeline["versions"]
+        ], f"Version {version} not found"
+    download_base_path = os.getenv(
+        "EOTDL_DOWNLOAD_PATH", str(Path.home()) + "/.cache/eotdl/pipelines"
+    )
+    if path is None:
+        download_path = download_base_path + "/" + pipeline_name
+    else:
+        download_path = path + "/" + pipeline_name
+    # check if pipeline already exists
+    if os.path.exists(download_path) and not force:
+        os.makedirs(download_path, exist_ok=True)
+        # raise Exception(
+        #     f"pipeline `{pipeline['name']} v{str(version)}` already exists at {download_path}. To force download, use force=True or -f in the CLI."
+        # )
+    
+    # stage metadata
+    repo = FilesAPIRepo()
+    catalog_path = repo.stage_file(pipeline["id"], f"catalog.v{version}.parquet", user, download_path)
+
+    # stage README.md
+    metadata = Metadata(**pipeline['metadata'], name=pipeline['name'])
+    metadata.save_metadata(download_path)
+
+    if assets:
+        gdf = gpd.read_parquet(catalog_path)
+        for _, row in tqdm(gdf.iterrows(), total=len(gdf), desc="Staging assets"):
+            for k, v in row["assets"].items():
+                stage_pipeline_file(v["href"], download_path)
+
+    return download_path
+
+@with_auth
+def stage_pipeline_file(file_url, path, user):
+    repo = FilesAPIRepo()
+    return repo.stage_file_url(file_url, path, user)

eotdl/fe/update.py

@@ -0,0 +1,12 @@
+from ..repos import FEAPIRepo
+from ..auth import with_auth
+from .retrieve import retrieve_pipeline
+
+@with_auth
+def deactivate_pipeline(pipeline_name, user):
+    pipeline = retrieve_pipeline(pipeline_name)
+    repo = FEAPIRepo()
+    data, error = repo.deactivate_pipeline(pipeline['id'], user)
+    if error:
+        raise Exception(error)
+    return data

eotdl/files/__init__.py

@@ -0,0 +1 @@
+from .get_url import get_file_url

eotdl/files/get_url.py

@@ -0,0 +1,18 @@
+from ..auth import with_auth
+from ..repos import FilesAPIRepo
+from ..datasets import retrieve_dataset
+from ..models import retrieve_model
+from ..fe import retrieve_pipeline
+
+@with_auth
+def get_file_url(filename, dataset_or_model_name, endpoint, user):
+    if endpoint == "datasets":
+        dataset_or_model_id = retrieve_dataset(dataset_or_model_name)['id']
+    elif endpoint == "models":
+        dataset_or_model_id = retrieve_model(dataset_or_model_name)['id']
+    elif endpoint == "pipelines":
+        dataset_or_model_id = retrieve_pipeline(dataset_or_model_name)['id']
+    else:
+        raise Exception("Invalid endpoint (datasets, models or pipelines)")
+    repo = FilesAPIRepo()
+    return repo.generate_presigned_url(filename, dataset_or_model_id, user, endpoint)

eotdl/files/ingest.py

@@ -98,13 +98,13 @@ def ingest_virtual( # could work for a list of paths with minimal changes...
 	data.append(create_stac_item('README.md', str(path / "README.md")))
 	gdf = gpd.GeoDataFrame(data, geometry='geometry')
 	gdf.to_parquet(path / "catalog.parquet")
-	return ingest(path, repo, retrieve, mode)
+	return ingest(path, repo, retrieve, mode, private=False)
 
 def ingest_catalog(path, repo, retrieve, mode):
-	return ingest(path, repo, retrieve, mode)
+	return ingest(path, repo, retrieve, mode, private=False)
 
 @with_auth
-def ingest(path, repo, retrieve, mode, user):
+def ingest(path, repo, retrieve, mode, private, user):
 	try:
 		readme = frontmatter.load(path.joinpath("README.md"))
 		metadata_dict = readme.metadata
@@ -115,7 +115,7 @@ def ingest(path, repo, retrieve, mode, user):
 		print(str(e))
 		raise Exception("Error loading metadata")
 	# retrieve dataset (create if doesn't exist)
-	dataset_or_model = retrieve(metadata, user)
+	dataset_or_model = retrieve(metadata, user, private)
 	current_version = sorted([v['version_id'] for v in dataset_or_model["versions"]])[-1]
 	# TODO: update README if metadata changed in UI (db)
 	# update_metadata = True

eotdl/files/metadata.py

@@ -35,7 +35,8 @@ class Metadata(BaseModel):
             f.write(f"name: {self.name}\n")
             f.write(f"license: {self.license}\n") 
             f.write(f"source: {self.source}\n")
-            f.write(f"thumbnail: {self.thumbnail}\n")
+            if self.thumbnail:
+                f.write(f"thumbnail: {self.thumbnail}\n")
             f.write(f"authors:\n")
             for author in self.authors:
                 f.write(f"  - {author}\n")

eotdl/models/ingest.py

@@ -3,7 +3,7 @@ from pathlib import Path
 from ..repos import ModelsAPIRepo
 from ..files.ingest import prep_ingest_stac, prep_ingest_folder, ingest, ingest_virtual, ingest_catalog
 
-def retrieve_model(metadata, user):
+def retrieve_model(metadata, user, private=False):
 	repo = ModelsAPIRepo()
 	data, error = repo.retrieve_model(metadata.name)
 	# print(data, error)
@@ -22,6 +22,7 @@ def ingest_model(
 	logger=print,
 	force_metadata_update=False,
 	sync_metadata=False,
+	private=False,
 ):
 	path = Path(path)
 	if not path.is_dir():
@@ -30,7 +31,7 @@ def ingest_model(
 		prep_ingest_stac(path, logger)
 	else:
 		prep_ingest_folder(path, verbose, logger, force_metadata_update, sync_metadata)
-	return ingest(path, ModelsAPIRepo(), retrieve_model, 'models')
+	return ingest(path, ModelsAPIRepo(), retrieve_model, 'models', private)
 
 def ingest_virtual_model( # could work for a list of paths with minimal changes...
 	path,

eotdl/repos/DatasetsAPIRepo.py

@@ -18,15 +18,26 @@ class DatasetsAPIRepo(APIRepo):
         response = requests.get(url)
         return self.format_response(response)
     
+    def retrieve_private_datasets(self, user):
+        url = self.url + "datasets/private"
+        response = requests.get(url, headers=self.generate_headers(user))
+        return self.format_response(response)
+    
     def retrieve_dataset(self, name):
         response = requests.get(self.url + "datasets?name=" + name)
         return self.format_response(response)
     
+    def retrieve_private_dataset(self, name, user):
+        response = requests.get(self.url + "datasets/private?name=" + name, headers=self.generate_headers(user))
+        return self.format_response(response)
+    
     def get_dataset_by_id(self, dataset_id):
         response = requests.get(self.url + "datasets/" + dataset_id)
         return self.format_response(response)
     
-    def create_dataset(self, metadata, user):
+    def create_dataset(self, metadata, user, private=False):
+        if private:
+            metadata["visibility"] = "private"
         response = requests.post(
             self.url + "datasets",
             json=metadata,