eotdl 2025.4.22__tar.gz → 2025.4.22.post2__tar.gz

{eotdl-2025.4.22 → eotdl-2025.4.22.post2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eotdl
-Version: 2025.4.22
+Version: 2025.4.22.post2
 Summary: Earth Observation Training Data Lab
 Author-email: earthpulse <it@earthpulse.es>
 License-Expression: MIT

eotdl-2025.4.22.post2/eotdl/__init__.py

@@ -0,0 +1 @@
+__version__ = "2025.04.22-2"

{eotdl-2025.4.22 → eotdl-2025.4.22.post2}/eotdl/commands/datasets.py

@@ -5,6 +5,7 @@ from ..datasets import (
     retrieve_datasets,
     ingest_dataset,
     stage_dataset,
+    deactivate_dataset,
 )
 
 app = typer.Typer(help="Explore, ingest and download training datasets.")
@@ -141,5 +142,15 @@ def get(
         typer.echo(e)
 
 
+@app.command()
+def deactivate(
+    dataset_id: str = typer.Argument(None, help="ID of the dataset to deactivate")
+):
+    try:
+        deactivate_dataset(dataset_id)
+        typer.echo(f"Dataset {dataset_id} deactivated")
+    except Exception as e:
+        typer.echo(e)
+
 if __name__ == "__main__":
     app()

{eotdl-2025.4.22 → eotdl-2025.4.22.post2}/eotdl/commands/models.py

@@ -5,6 +5,7 @@ from ..models import (
     retrieve_models,
     ingest_model,
     stage_model,
+    deactivate_model
 )
 
 app = typer.Typer(help="Explore, ingest and download ML models.")
@@ -132,5 +133,16 @@ def get(
         typer.echo(e)
 
 
+@app.command()
+def deactivate(
+    model_id: str = typer.Argument(None, help="ID of the model to deactivate")
+):
+    try:
+        deactivate_model(model_id)
+        typer.echo(f"Model {model_id} deactivated")
+    except Exception as e:
+        typer.echo(e)
+
+
 if __name__ == "__main__":
     app()

{eotdl-2025.4.22 → eotdl-2025.4.22.post2}/eotdl/commands/stac.py

@@ -24,9 +24,9 @@ def collections():
         raise typer.Abort()
     
 @app.command()
-def collection(collection_id: str):
+def collection(collection_name: str):
     try:
-        data = retrieve_stac_collection(collection_id)
+        data = retrieve_stac_collection(collection_name)
         typer.echo(data)
     except Exception as e:
         typer.echo(e)

{eotdl-2025.4.22 → eotdl-2025.4.22.post2}/eotdl/curation/stac/api.py

@@ -16,9 +16,9 @@ def retrieve_stac_collections():
         raise Exception(error)
     return data
 
-def retrieve_stac_collection(collection_id):
+def retrieve_stac_collection(collection_name):
     repo = STACAPIRepo()
-    data, error = repo.collection(collection_id)
+    data, error = repo.collection(collection_name)
     if error:
         raise Exception(error)
     return data

{eotdl-2025.4.22 → eotdl-2025.4.22.post2}/eotdl/datasets/__init__.py

@@ -1,3 +1,4 @@
 from .retrieve import retrieve_datasets, retrieve_dataset, retrieve_dataset_files
 from .ingest import ingest_dataset, ingest_virtual_dataset, ingest_dataset_catalog
 from .stage import stage_dataset, stage_dataset_file
+from .update import deactivate_dataset

{eotdl-2025.4.22 → eotdl-2025.4.22.post2}/eotdl/datasets/update.py

@@ -15,3 +15,11 @@ def update_dataset(dataset_id, metadata, content, user):
     if error:
         raise Exception(error)
     return data
+
+
+def deactivate_dataset(dataset_id):
+    repo = DatasetsAPIRepo()
+    data, error = repo.deactivate_dataset(dataset_id)
+    if error:
+        raise Exception(error)
+    return data

{eotdl-2025.4.22 → eotdl-2025.4.22.post2}/eotdl/models/__init__.py

@@ -1,3 +1,4 @@
 from .retrieve import retrieve_models, retrieve_model, retrieve_model_files
 from .ingest import ingest_model, ingest_virtual_model, ingest_model_catalog
 from .stage import stage_model, stage_model_file
+from .update import deactivate_model

{eotdl-2025.4.22 → eotdl-2025.4.22.post2}/eotdl/models/update.py

@@ -15,3 +15,10 @@ def update_model(model_id, metadata, content, user):
     if error:
         raise Exception(error)
     return data
+
+def deactivate_model(model_id):
+    repo = ModelsAPIRepo()
+    data, error = repo.deactivate_model(model_id)
+    if error:
+        raise Exception(error)
+    return data

{eotdl-2025.4.22 → eotdl-2025.4.22.post2}/eotdl/repos/APIRepo.py

@@ -4,8 +4,8 @@ import requests
 
 class APIRepo:
     def __init__(self, url=None):
-        default_url = "https://api.eotdl.com/"
-        # default_url = "http://localhost:8000/"
+        # default_url = "https://api.eotdl.com/"
+        default_url = "http://localhost:8000/"
         self.url = url if url else os.getenv("EOTDL_API_URL", default_url)
 
     def format_response(self, response):

{eotdl-2025.4.22 → eotdl-2025.4.22.post2}/eotdl/repos/DatasetsAPIRepo.py

@@ -41,3 +41,9 @@ class DatasetsAPIRepo(APIRepo):
             headers=self.generate_headers(user),
         )
         return self.format_response(response)
+    
+    def deactivate_dataset(self, dataset_name):
+        response = requests.patch(
+            self.url + "datasets/deactivate/" + dataset_name
+        )
+        return self.format_response(response)

{eotdl-2025.4.22 → eotdl-2025.4.22.post2}/eotdl/repos/ModelsAPIRepo.py

@@ -38,4 +38,8 @@ class ModelsAPIRepo(APIRepo):
         )
         return self.format_response(response)
 
-    
+    def deactivate_model(self, model_name):
+        response = requests.patch(
+            self.url + "models/deactivate/" + model_name
+        )
+        return self.format_response(response)

{eotdl-2025.4.22 → eotdl-2025.4.22.post2}/eotdl/repos/STACAPIRepo.py

@@ -15,8 +15,8 @@ class STACAPIRepo(APIRepo):
         response = requests.get(self.url + "stac/collections")
         return self.format_response(response)
 
-    def collection(self, collection_id):
-        response = requests.get(self.url + f"stac/collections/{collection_id}")
+    def collection(self, collection_name):
+        response = requests.get(self.url + f"stac/collections/{collection_name}")
         return self.format_response(response)
 
     def items(self, collection_id):

{eotdl-2025.4.22 → eotdl-2025.4.22.post2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "eotdl"
-version = "2025.04.22"
+version = "2025.04.22-2"
 description = "Earth Observation Training Data Lab"
 authors = [
     {name = "earthpulse", email = "it@earthpulse.es"},

eotdl-2025.4.22.post2/tests/load/conftest.py

@@ -0,0 +1,55 @@
+import json
+from bson import ObjectId
+from minio import Minio
+from pymongo import MongoClient
+import pytest
+import boto3
+
+
+@pytest.fixture
+def setup_mongo():
+    client = MongoClient("mongodb://localhost:27017/")
+    db = client["eotdl-test"]
+
+    client.drop_database("eotdl-test")
+
+    db = client["eotdl-test"]
+    tiers_collection = db["tiers"]
+
+    with open("eotdl/tests/load/eotdl.tiers.json", "r") as file:
+        json_data = json.load(file)
+
+    for item in json_data:
+        if "_id" in item:
+            item["_id"] = ObjectId(item["_id"])
+
+    tiers_collection.insert_many(json_data)
+
+    yield tiers_collection
+
+    client.drop_database("eotdl-test")
+
+
+@pytest.fixture
+def setup_minio():
+    minio_client = Minio(
+        "localhost:9000",
+        access_key="eotdl",
+        secret_key="12345678",
+        secure=False, 
+    )
+
+    bucket_name = "eotdl-test"
+
+    if not minio_client.bucket_exists(bucket_name):
+        minio_client.make_bucket(bucket_name)
+
+    objects = minio_client.list_objects(bucket_name, recursive=True)
+    for obj in objects:
+        minio_client.remove_object(bucket_name, obj.object_name)
+
+    yield minio_client
+
+    objects = minio_client.list_objects(bucket_name, recursive=True)
+    for obj in objects:
+        minio_client.remove_object(bucket_name, obj.object_name)

eotdl-2025.4.22.post2/tests/load/eotdl.tiers.json

@@ -0,0 +1,34 @@
+[
+  {
+    "_id": "645242248456b2cc058e43bf",
+    "name": "free",
+    "limits": {
+      "datasets": {
+        "upload": 10,
+        "download": 3,
+        "count": 10
+      },
+      "models": {
+        "upload": 10,
+        "download": 10,
+        "count": 10
+      }
+    }
+  },
+  {
+    "_id": "645242248456b2cc058e43c0",
+    "name": "dev",
+    "limits": {
+      "datasets": {
+        "upload": 100000,
+        "download": 100000,
+        "count": 100
+      },
+      "models": {
+        "upload": 100000,
+        "download": 100000,
+        "count": 100
+      }
+    }
+  }
+]

eotdl-2025.4.22.post2/tests/load/test_upload_size.py

@@ -0,0 +1,194 @@
+from datetime import datetime, timezone
+import io
+import os
+from pathlib import Path
+import tempfile
+import time
+
+from eotdl.datasets.retrieve import retrieve_dataset_files
+import numpy as np
+import pytest
+
+from eotdl.datasets import ingest_dataset, retrieve_dataset
+import rasterio
+
+
+os.environ["EOTDL_API_URL"] = "http://localhost:8000/"
+
+
+def write_readme(path: Path, name: str):
+    readme_text = f"""---
+name: {name}
+authors: 
+  - Derek van de Ven
+license: free
+source: https://github.com/earthpulse/eotdl/blob/main/tutorials/notebooks/02_ingesting.ipynb
+---
+
+# {name}
+
+This file is nonsensical data used for load testing. It should not be stored on the web. 
+"""
+
+    with open(path / name / "README.md", "w") as outfile:
+        outfile.write(readme_text)
+
+
+def generate_fake_tif_dataset(
+    path: Path, n_tifs: int = 10, tif_size: int = 256, name: str = "test_set"
+):
+    files_to_upload = []
+    for count in range(n_tifs):
+        files_to_upload.append(f"{name}/FakeFolder/Fake_tif{count}.tif")
+
+    for rel_path in files_to_upload:
+        file_path = path / rel_path
+        file_path.parent.mkdir(parents=True, exist_ok=True)
+
+        width, height = tif_size, tif_size
+        data = np.full((height, width, 3), 256, dtype=np.uint32)
+
+        with rasterio.open(
+            file_path,
+            "w",
+            driver="GTiff",
+            height=height,
+            width=width,
+            count=3,
+            dtype="uint32",
+            crs="EPSG:4326",
+            transform=rasterio.transform.from_origin(0, height, 1, 1),
+        ) as dst:
+            dst.write(data[:, :, 0], 1)
+            dst.write(data[:, :, 1], 2)
+            dst.write(data[:, :, 2], 3)
+
+    write_readme(path, name)
+
+
+def generate_fake_dataset(
+    path: Path, size_mb: int = 10, n_files: int = 5, name: str = "test_set"
+):
+    files_to_upload = []
+    for count in range(n_files):
+        files_to_upload.append(
+            f"{name}/FakeFolder/Fake_tif{count}.tif",
+        )
+
+    total_size_bytes = size_mb * 1024 * 1024
+    file_size = total_size_bytes // len(files_to_upload)
+
+    for rel_path in files_to_upload:
+        file_path = path / rel_path
+        file_path.parent.mkdir(parents=True, exist_ok=True)
+
+        # Write random binary data to simulate `.tif` files
+        with open(file_path, "wb") as f:
+            f.write(os.urandom(file_size))  # Random binary data
+
+    write_readme(path, name)
+
+
+@pytest.mark.parametrize(
+    "total_size, n_files",
+    [
+        (1, 10),
+        # (1e1, 10),
+        # (1e3, 1), # 1 files amounting to 1 GB,
+        # (1e3, 20), # 20 files amounting to 1 GB
+        # (1e4, 100), # 100 10 MB files
+        # (3e4, 100), # 10 times 1 GB
+    ],
+)
+def test_load(setup_mongo, setup_minio, total_size, n_files):
+    name = f"LoadTest-{int(total_size)}MB"
+    with tempfile.TemporaryDirectory(prefix="loadtest_") as tmpdir:
+        tmpdir = Path(tmpdir)
+        generate_fake_dataset(
+            path=tmpdir, size_mb=int(total_size), n_files=n_files, name=name
+        )
+
+        # upload
+        start_time = time.time()
+        ingest_dataset(path=str(tmpdir / name))
+
+        upload_duration = time.time() - start_time
+        print(f"Upload for {name} took {upload_duration:.2f} seconds.")
+
+        with open("eotdl/tests/load/upload_times.log", "a") as log_file:
+            log_file.write(
+                f"{total_size} MB made of {n_files} files took {upload_duration:.2f} seconds.\n"
+            )
+
+        # assert dataset ingested
+        dataset = retrieve_dataset(name=name)
+        assert dataset["name"] == name
+        assert round(dataset["versions"][0]["size"] / (1024 * 1024), 2) == total_size
+
+        # check files in minio
+        minio_files = []
+        minio_sizes = []
+        for obj in setup_minio.list_objects("eotdl-test", recursive=True):
+            minio_files.append(obj.object_name)
+            minio_sizes.append(obj.size)
+            assert obj.object_name.endswith(("tif", "md", "parquet"))
+
+        assert len(minio_files) == n_files + 2
+        assert round(sum(minio_sizes) / (1024 * 1024)) == total_size
+
+
+@pytest.mark.parametrize(
+    "tif_size, n_tifs",
+    [
+        # (256, 100),
+        # (512, 100),
+        # (1024, 100),
+        (2048, 200),
+    ],
+)
+def test_load_tifs(setup_mongo, setup_minio, tif_size, n_tifs):
+    name = f"LoadTest-{int(tif_size)}"
+    with tempfile.TemporaryDirectory(prefix="loadtest_") as tmpdir:
+        tmpdir = Path(tmpdir)
+        generate_fake_tif_dataset(
+            path=tmpdir, tif_size=tif_size, n_tifs=n_tifs, name=name
+        )
+
+        # upload
+        start_time = time.time()
+        ingest_dataset(path=str(tmpdir / name))
+
+        upload_duration = time.time() - start_time
+        print(f"Upload for {name} took {upload_duration:.2f} seconds.")
+
+        # assert dataset ingested
+        dataset = retrieve_dataset(name=name)
+        assert dataset["name"] == name
+
+        with open("eotdl/tests/load/upload_times_tifs.log", "a") as log_file:
+            log_file.write(
+                f"{tif_size}x {tif_size} made of {n_tifs} files took {upload_duration:.2f} seconds. ({round(dataset['versions'][0]['size'] / (1024 * 1024), 2)} mb)\n"
+            )
+
+        # check files in minio
+        minio_files = []
+        minio_sizes = []
+        for obj in setup_minio.list_objects("eotdl-test", recursive=True):
+            minio_files.append(obj.object_name)
+            minio_sizes.append(obj.size)
+            assert obj.object_name.endswith(("tif", "md", "parquet"))
+
+            if obj.object_name.endswith(".tif"):
+                file = setup_minio.get_object("eotdl-test", obj.object_name)
+                file_content = file.read()
+                with io.BytesIO(file_content) as byte_file:
+                    with rasterio.open(byte_file) as src:
+                        assert src.crs == "EPSG:4326"
+                        assert src.width == src.height == tif_size
+                        assert src.dtypes == ("uint32", "uint32", "uint32")
+
+                        for band in range(1, 4):
+                            colour = src.read(band)
+                            assert set(colour.flatten()) == {np.uint32(256)}
+
+        assert len(minio_files) == n_tifs + 2

eotdl-2025.4.22.post2/tests/load/upload_times.log

@@ -0,0 +1,6 @@
+1 MB made of 10 files took 0.46 seconds.
+10.0 MB made of 10 files took 0.44 seconds.
+1000.0 MB made of 1 files took 9.78 seconds.
+1000.0 MB made of 20 files took 10.35 seconds.
+10000.0 MB made of 100 files took 120.89 seconds.
+10000.0 MB made of 10 files took 133.24 seconds.

eotdl-2025.4.22.post2/tests/load/upload_times_tifs.log

@@ -0,0 +1,5 @@
+256x 256 made of 100 files took 5.89 seconds. (75.11 mb)
+512x 512 made of 100 files took 8.48 seconds. (300.33 mb)
+1024x 1024 made of 100 files took 23.38 seconds. (1200.62 mb)
+2048x 2048 made of 100 files took 81.27 seconds. (4801.21 mb)
+2048x 2048 made of 200 files took 202.64 seconds. (9602.42 mb)

{eotdl-2025.4.22 → eotdl-2025.4.22.post2}/uv.lock

@@ -110,7 +110,7 @@ wheels = [
 
 [[package]]
 name = "eotdl"
-version = "2025.4.2.post4"
+version = "2025.4.22"
 source = { editable = "." }
 dependencies = [
     { name = "geopandas" },

eotdl-2025.4.22/eotdl/__init__.py

@@ -1 +0,0 @@
-__version__ = "2025.04.22"

eotdl-2025.4.22/pyproject.toml.bck

@@ -1,35 +0,0 @@
-[project]
-name = "eotdl"
-version = "2025.04.02"
-description = "Earth Observation Training Data Lab"
-authors = [
-    {name = "earthpulse", email = "it@earthpulse.es"},
-]
-license = "MIT"
-readme = "README.md"
-requires-python = ">=3.8"
-dependencies = [
-    "geomet>=1.1.0",
-    "geopandas>=0.13.2",
-    "markdown>=3.7",
-    "pydantic>=2.10.6",
-    "pyjwt>=2.9.0",
-    "pystac>=1.8.4",
-    "python-frontmatter>=1.1.0",
-    "pyyaml>=6.0.2",
-    "rasterio>=1.3.11",
-    "requests>=2.32.3",
-    "sentinelhub>=3.11.1",
-    "tqdm>=4.67.1",
-    "typer>=0.15.1",
-]
-
-[project.scripts]
-eotdl = "eotdl.cli:app"
-
-[build-system]
-requires = ["hatchling"]
-build-backend = "hatchling.build"
-
-[tool.hatch.build.targets.wheel]
-packages = ["eotdl"]