eotdl 2024.10.7__tar.gz → 2025.2.10__tar.gz

eotdl-2025.2.10/.gitignore

@@ -0,0 +1,40 @@
+__pycache__
+.pytest_cache
+dist
+.DS_Store
+node_modules
+build
+.svelte-kit
+package
+.env
+.env.*
+.conda
+!.env.example
+yarn.lock
+poetry.lock
+.vercel
+data
+metadata
+eurosat-stac
+.ipynb_checkpoints
+eurosat-stac**
+.coverage
+data
+db
+*.zip
+docker-compose.prod.yml
+tutorials/*md
+!test.zip
+.vscode
+kk
+auth.json
+node_modules/
+/test-results/
+playwright-report
+playwright/.cache/
+test-results
+lightning_logs
+jobs.csv
+*.parquet
+download_commands.txt
+_data

{eotdl-2024.10.7 → eotdl-2025.2.10}/PKG-INFO

@@ -1,35 +1,23 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: eotdl
-Version: 2024.10.7
+Version: 2025.2.10
 Summary: Earth Observation Training Data Lab
-License: MIT
-Author: EarthPulse
-Author-email: it@earthpulse.es
-Requires-Python: >=3.8,<4.0
-Classifier: License :: OSI Approved :: MIT License
-Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.8
-Classifier: Programming Language :: Python :: 3.9
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.12
-Requires-Dist: black (>=23.10.1,<24.0.0)
-Requires-Dist: geomet (>=1.0.0,<2.0.0)
-Requires-Dist: geopandas (>=0.13.2,<0.14.0)
-Requires-Dist: markdown (>=3.5.2,<4.0.0)
-Requires-Dist: markdownify (>=0.11.6,<0.12.0)
-Requires-Dist: mypy (>=1.6.1,<2.0.0)
-Requires-Dist: openeo (>=0.31.0,<0.32.0)
-Requires-Dist: pydantic (>=1.10.6,<2.0.0)
-Requires-Dist: pyjwt (>=2.6.0,<3.0.0)
-Requires-Dist: pystac[validation] (==1.8.2)
-Requires-Dist: python-frontmatter (>=1.1.0,<2.0.0)
-Requires-Dist: pyyaml (>=6.0.1,<7.0.0)
-Requires-Dist: rasterio (>=1.3.9,<2.0.0)
-Requires-Dist: requests (>=2.28.2,<3.0.0)
-Requires-Dist: sentinelhub (>=3.9.1,<4.0.0)
-Requires-Dist: tqdm (>=4.65.0,<5.0.0)
-Requires-Dist: typer[all] (>=0.9.0,<0.10.0)
+Author-email: earthpulse <it@earthpulse.es>
+License-Expression: MIT
+Requires-Python: >=3.8
+Requires-Dist: geomet>=1.1.0
+Requires-Dist: geopandas>=0.13.2
+Requires-Dist: markdown>=3.7
+Requires-Dist: pydantic>=2.10.6
+Requires-Dist: pyjwt>=2.9.0
+Requires-Dist: pystac>=1.8.4
+Requires-Dist: python-frontmatter>=1.1.0
+Requires-Dist: pyyaml>=6.0.2
+Requires-Dist: rasterio>=1.3.11
+Requires-Dist: requests>=2.32.3
+Requires-Dist: sentinelhub>=3.11.1
+Requires-Dist: tqdm>=4.67.1
+Requires-Dist: typer>=0.15.1
 Description-Content-Type: text/markdown
 
 <p align="center">
@@ -49,10 +37,10 @@ Description-Content-Type: text/markdown
 
 This is the main library and CLI for the **Earth Observation Training Data Lab** (EOTDL), a complete environment that allows you, among other things, to:
 
-- Explore and download Training Datasets (TDS) for Earth Observation (EO) applications.
+- Explore and stage Training Datasets (TDS) for Earth Observation (EO) applications.
 - Create and upload your own TDS by combining and annotating EO data from different sources.
 - Train Machine Learning (ML) models using the hosted TDS in the cloud with multi-GPU machines.
-- Explore and download pre-trianed ML models for EO applications.
+- Explore and stage pre-trianed ML models for EO applications.
 
 In our blog you will find tutorials to learn how leverage the EOTDL to create and use TDS and ML models for your own EO applications.
 
@@ -62,4 +50,4 @@ One of the most limiting factors of AI for EO applications is the scarcity of su
 
 Generating TDS is time consuming and expensive. Data access is usually limited and costly, especially for Very High Resolution (VHR) images that allow objects like trees to be clearly identified. In some cases, domain experts or even in-person (in-situ) trips are required to manually confirm the objects in a satellite image are correctly annotated with a high degree of quality. This results in the field of AI for EO applications lagging when compared to other fields, impeding the development of new applications and limiting the full potential of AI in EO.
 
-The European Space Agency (ESA) Earth Observation Training Data Lab (EOTDL) will address key limitations and capability gaps for working with Machine Learning (ML) training data in EO by providing a set of open-source tools to create, share, and improve datasets as well as training ML algorithms in the cloud. EOTDL will also offer an online repository where datasets and models can be explored and accessed.
+The European Space Agency (ESA) Earth Observation Training Data Lab (EOTDL) will address key limitations and capability gaps for working with Machine Learning (ML) training data in EO by providing a set of open-source tools to create, share, and improve datasets as well as training ML algorithms in the cloud. EOTDL will also offer an online repository where datasets and models can be explored and accessed.

{eotdl-2024.10.7 → eotdl-2025.2.10}/README.md

@@ -15,10 +15,10 @@
 
 This is the main library and CLI for the **Earth Observation Training Data Lab** (EOTDL), a complete environment that allows you, among other things, to:
 
-- Explore and download Training Datasets (TDS) for Earth Observation (EO) applications.
+- Explore and stage Training Datasets (TDS) for Earth Observation (EO) applications.
 - Create and upload your own TDS by combining and annotating EO data from different sources.
 - Train Machine Learning (ML) models using the hosted TDS in the cloud with multi-GPU machines.
-- Explore and download pre-trianed ML models for EO applications.
+- Explore and stage pre-trianed ML models for EO applications.
 
 In our blog you will find tutorials to learn how leverage the EOTDL to create and use TDS and ML models for your own EO applications.
 

eotdl-2025.2.10/eotdl/__init__.py

@@ -0,0 +1 @@
+__version__ = "2025.02.10"

{eotdl-2024.10.7 → eotdl-2025.2.10}/eotdl/access/download.py

@@ -42,8 +42,9 @@ def download_sentinel_imagery(
         bulk = False
     else:
         bulk = True
-    client.download_data(requests_list)
-    imagery_from_tmp_to_dir(output, name=name, bulk=bulk)
+
+    data = client.download_data(requests_list)
+    imagery_from_tmp_to_dir(output, client.tmp_dir, name=name, bulk=bulk)
 
 
 def search_and_download_sentinel_imagery(

{eotdl-2024.10.7 → eotdl-2025.2.10}/eotdl/access/search.py

@@ -19,8 +19,6 @@ def search_sentinel_imagery(
     evaluate_sentinel_parameters(
         sensor, time_interval, bounding_box, output_needed=False
     )
-
     client = SHClient()
     parameters = SH_PARAMETERS_DICT[sensor]()
-
     return client.search_data(bounding_box, time_interval, parameters)

{eotdl-2024.10.7 → eotdl-2025.2.10}/eotdl/access/sentinelhub/client.py

@@ -15,6 +15,7 @@ from sentinelhub import (
     SentinelHubDownloadClient,
     MimeType,
 )
+import uuid
 
 from ...repos.AuthRepo import AuthRepo
 from .parameters import SHParameters
@@ -57,7 +58,7 @@ class SHClient:
                     self.config.sh_client_id = creds["SH_CLIENT_ID"]
                     self.config.sh_client_secret = creds["SH_CLIENT_SECRET"]
         self.catalog = SentinelHubCatalog(config=self.config)
-        self.tmp_dir = "/tmp/sentinelhub"
+        self.tmp_dir = "/tmp/sentinelhub/" + str(uuid.uuid4())
 
     def search_data(
         self, bounding_box: list, time_interval: list, parameters: SHParameters
@@ -109,5 +110,4 @@ class SHClient:
             requests = [requests]
         download_requests = [request.download_list[0] for request in requests]
         data = download_client.download(download_requests)
-
         return data

{eotdl-2024.10.7 → eotdl-2025.2.10}/eotdl/access/sentinelhub/parameters.py

@@ -42,7 +42,7 @@ class SHS2L1CParameters(SHParameters):
         "include": ["id", "properties.datetime", "properties.eo:cloud_cover"],
         "exclude": [],
     }
-
+    FILTER = None
 
 class SHS1Parameters(SHParameters):
     """

{eotdl-2024.10.7 → eotdl-2025.2.10}/eotdl/access/sentinelhub/utils.py

@@ -50,7 +50,7 @@ def evaluate_sentinel_parameters(
 
 def imagery_from_tmp_to_dir(
     output_dir: str,
-    tmp_dir: Optional[str] = "/tmp/sentinelhub",
+    tmp_dir: Optional[str],
     name: Optional[str] = None,
     bulk: Optional[bool] = False,
 ) -> None:
@@ -60,9 +60,7 @@ def imagery_from_tmp_to_dir(
     downloaded_files = glob(f"{tmp_dir}/**/response.tiff")
     if len(downloaded_files) == 0:
         return
-
     makedirs(output_dir, exist_ok=True)
-
     for downloaded_file in downloaded_files:
         request_json = downloaded_file.replace("response.tiff", "request.json")
         metadata = generate_raster_metadata(downloaded_file, request_json)
@@ -75,11 +73,9 @@ def imagery_from_tmp_to_dir(
                 output_filename = f"{metadata['type']}_{metadata['acquisition-date']}"
             else:
                 output_filename = metadata["type"]
-
         copyfile(downloaded_file, f"{output_dir}/{output_filename}.tif")
         with open(f"{output_dir}/{output_filename}.json", "w", encoding="utf-8") as f:
             json.dump(metadata, f)
-
     rmtree(tmp_dir)
 
 

{eotdl-2024.10.7 → eotdl-2025.2.10}/eotdl/curation/stac/extensions/ml_dataset.py

@@ -377,7 +377,7 @@ class MLDatasetQualityMetrics:
                 destination
             )  # Remove the old catalog and replace it with the new one
             catalog.set_root(catalog)
-            catalog.normalize_and_save(root_href=destination)
+            catalog.normalize_and_save(root_href=destination, catalog_type=pystac.CatalogType.SELF_CONTAINED)
             print("Success!")
         except STACValidationError:
             # Return full callback

{eotdl-2024.10.7 → eotdl-2025.2.10}/eotdl/datasets/ingest.py

@@ -3,6 +3,7 @@ import yaml
 from tqdm import tqdm
 import json
 import frontmatter
+import pystac
 
 from ..auth import with_auth
 from .metadata import Metadata
@@ -129,7 +130,7 @@ def ingest_stac(stac_catalog, logger=None, user=None):
     repo, files_repo = DatasetsAPIRepo(), FilesAPIRepo()
     # load catalog
     logger("Loading STAC catalog...")
-    df = STACDataFrame.from_stac_file(stac_catalog)
+    df = STACDataFrame.from_stac_file(stac_catalog) # assets are absolute for file ingestion
     catalog = df[df["type"] == "Catalog"]
     assert len(catalog) == 1, "STAC catalog must have exactly one root catalog"
     dataset_name = catalog.id.iloc[0]

{eotdl-2024.10.7 → eotdl-2025.2.10}/eotdl/models/download.py

@@ -74,12 +74,12 @@ def download_model(
         )
         if error:
             raise Exception(error)
-        print(gdf)
+        # print(gdf)
         df = STACDataFrame(gdf)
         # df.geometry = df.geometry.apply(lambda x: Polygon() if x is None else x)
         df.to_stac(download_path)
-        print("----")
-        print(df)
+        # print("----")
+        # print(df)
         # download assets
         if assets:
             if verbose:

{eotdl-2024.10.7 → eotdl-2025.2.10}/eotdl/repos/APIRepo.py

@@ -5,7 +5,7 @@ import requests
 class APIRepo:
     def __init__(self, url=None):
         default_url = "https://api.eotdl.com/"
-        # default_url = "http://localhost:8010/"
+        # default_url = "http://localhost:8001/"
         self.url = url if url else os.getenv("EOTDL_API_URL", default_url)
 
     def format_response(self, response):

{eotdl-2024.10.7 → eotdl-2025.2.10}/eotdl/tools/geo_utils.py

@@ -127,6 +127,11 @@ def bbox_from_centroid(
     width_m = width * pixel_size
     heigth_m = height * pixel_size
 
+    # Initialise the transformers
+    utm_crs = CRS.get_utm_from_wgs84(y, x).ogc_string()
+    from_4326_transformer = Transformer.from_crs("EPSG:4326", utm_crs)
+    to_4326_transformer = Transformer.from_crs(utm_crs, "EPSG:4326")
+
     # Transform the centroid coordinates to meters
     centroid_m = from_4326_transformer.transform(x, y)
 
@@ -137,8 +142,8 @@ def bbox_from_centroid(
     max_y = centroid_m[1] + heigth_m / 2
 
     # Convert the bounding box coordinates back to degrees
-    min_x, min_y = from_3857_transformer.transform(min_x, min_y)
-    max_x, max_y = from_3857_transformer.transform(max_x, max_y)
+    min_x, min_y = to_4326_transformer.transform(min_x, min_y)
+    max_x, max_y = to_4326_transformer.transform(max_x, max_y)
 
     return [min_y, min_x, max_y, max_x]
 

eotdl-2025.2.10/main.py

@@ -0,0 +1,5 @@
+from eotdl.cli import app
+
+if __name__ == "__main__":
+    app()
+

eotdl-2025.2.10/pyproject.toml

@@ -0,0 +1,35 @@
+[project]
+name = "eotdl"
+version = "2025.02.10"
+description = "Earth Observation Training Data Lab"
+authors = [
+    {name = "earthpulse", email = "it@earthpulse.es"},
+]
+license = "MIT"
+readme = "README.md"
+requires-python = ">=3.8"
+dependencies = [
+    "geomet>=1.1.0",
+    "geopandas>=0.13.2",
+    "markdown>=3.7",
+    "pydantic>=2.10.6",
+    "pyjwt>=2.9.0",
+    "pystac>=1.8.4",
+    "python-frontmatter>=1.1.0",
+    "pyyaml>=6.0.2",
+    "rasterio>=1.3.11",
+    "requests>=2.32.3",
+    "sentinelhub>=3.11.1",
+    "tqdm>=4.67.1",
+    "typer>=0.15.1",
+]
+
+[project.scripts]
+eotdl = "eotdl.cli:app"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["eotdl"]

eotdl-2025.2.10/tests/_test_access.py

@@ -0,0 +1,24 @@
+import pytest
+try:
+    from lib.eotdl import SHClient
+except ImportError:
+    from eotdl import SHClient
+
+
+def test_sh_client():
+    client = SHClient(sh_client_id='my_client_id',
+    sh_client_secret='my_client_secret')
+
+    assert client.config.sh_client_id is not None
+
+
+def test_search_available_sentinel_data():
+    pass
+
+
+def test_request_bulk_data():
+    pass
+
+
+def test_request_data():
+    pass

eotdl-2025.2.10/tests/_test_hello.py

@@ -0,0 +1,6 @@
+import pytest
+from eotdl.hello import say_hello
+
+
+def test_hello():
+    assert say_hello() == "Hello, World!"

eotdl-2025.2.10/tests/unit/_test_auth.py

@@ -0,0 +1,97 @@
+import pytest
+from unittest import mock
+
+from eotdl.eotdl.auth.auth import Auth
+from eotdl.src.errors.auth import LoginError, AuthTimeOut
+from eotdl.eotdl.auth.is_logged import IsLogged
+from eotdl.eotdl.auth.logout import Logout
+
+
+def test_auth():
+    repo = mock.Mock()
+    api_repo = mock.Mock()
+    auth = Auth(repo, api_repo)
+    inputs = Auth.Inputs()
+    api_repo.login.return_value = mock.Mock(
+        status_code=200,
+        json=mock.Mock(return_value={"code": 123, "login_url": "123123"}),
+    )
+    token_data = {"id_token": "lkajshdflkjahsldkfjhalskjdfhlkasjdhf"}
+    api_repo.token.return_value = mock.Mock(
+        status_code=200, json=mock.Mock(return_value=token_data)
+    )
+    repo.save_creds.return_value = "creds_path"
+    repo.decode_token.return_value = {
+        "uid": "test",
+        "name": "test name",
+        "email": "test email",
+        "picture": "test picture",
+    }
+    outputs = auth(inputs)
+    api_repo.login.assert_called_once()
+    api_repo.token.assert_called_once_with(123)
+    repo.save_creds.assert_called_once_with(token_data)
+    repo.decode_token.assert_called_once_with(token_data)
+    assert outputs.user["uid"] == "test"
+    assert outputs.user["name"] == "test name"
+    assert outputs.user["email"] == "test email"
+    assert outputs.user["picture"] == "test picture"
+
+
+def test_auth_fails_if_not_logged_in():
+    repo = mock.Mock()
+    api_repo = mock.Mock()
+    auth = Auth(repo, api_repo)
+    inputs = Auth.Inputs()
+    api_repo.login.return_value = mock.Mock(status_code=400)
+    with pytest.raises(LoginError):
+        auth(inputs)
+    api_repo.login.assert_called_once()
+
+
+def test_auth_fails_if_auth_timeout():
+    repo = mock.Mock()
+    api_repo = mock.Mock()
+    auth = Auth(
+        repo, api_repo, max_t=0.1, interval=0.01
+    )  # will try to login 10 times, then timeout
+    inputs = Auth.Inputs()
+    api_repo.login.return_value = mock.Mock(
+        status_code=200,
+        json=mock.Mock(return_value={"code": 123, "login_url": "123123"}),
+    )
+    api_repo.token.return_value = mock.Mock(status_code=400)
+    with pytest.raises(AuthTimeOut):
+        auth(inputs)
+
+
+def test_is_logged_in():
+    repo = mock.Mock()
+    repo.load_creds.return_value = {"uid": 123}
+    is_logged = IsLogged(repo)
+    inputs = IsLogged.Inputs()
+    outputs = is_logged(inputs)
+    repo.load_creds.assert_called_once()
+    assert outputs.user == {"uid": 123}
+
+
+def test_is_not_logged_in():
+    repo = mock.Mock()
+    repo.load_creds.return_value = None
+    is_logged = IsLogged(repo)
+    inputs = IsLogged.Inputs()
+    outputs = is_logged(inputs)
+    repo.load_creds.assert_called_once()
+    assert outputs.user is None
+
+
+def test_logout():
+    repo = mock.Mock()
+    api_repo = mock.Mock()
+    logout = Logout(repo, api_repo)
+    inputs = Logout.Inputs()
+    api_repo.logout_url.return_value = "ñklasjdf"
+    outputs = logout(inputs)
+    api_repo.logout_url.assert_called_once()
+    repo.logout.assert_called_once()
+    assert outputs.logout_url == "ñklasjdf"

eotdl-2025.2.10/tests/unit/curation/stac/fixtures.py

@@ -0,0 +1,76 @@
+import os
+import shutil
+import pytest
+
+from pystac import Catalog
+
+EXAMPLE_DATA_DIR = "tests/unit/curation/stac/test_data"
+
+
+@pytest.fixture
+def tmp_stac_catalog():
+    original_catalog_path = os.path.join(EXAMPLE_DATA_DIR, "jaca_dataset_stac")
+
+    os.makedirs("tmp", exist_ok=True)
+    tmp_catalog_path = os.path.join("tmp", "copied_catalog")
+    tmp_catalog = Catalog.from_file(os.path.join(original_catalog_path, "catalog.json"))
+    tmp_catalog.normalize_and_save(tmp_catalog_path)
+
+    copied_catalog = Catalog.from_file(os.path.join(tmp_catalog_path, "catalog.json"))
+    yield copied_catalog
+
+    shutil.rmtree("tmp")
+
+
+@pytest.fixture
+def tmp_stac_catalog_labels():
+    original_catalog_path = os.path.join(EXAMPLE_DATA_DIR, "jaca_dataset_stac_labels")
+
+    os.makedirs("tmp", exist_ok=True)
+    tmp_catalog_path = os.path.join("tmp", "copied_catalog")
+    tmp_catalog = Catalog.from_file(os.path.join(original_catalog_path, "catalog.json"))
+    tmp_catalog.normalize_and_save(tmp_catalog_path)
+
+    copied_catalog = Catalog.from_file(os.path.join(tmp_catalog_path, "catalog.json"))
+    yield copied_catalog
+
+    shutil.rmtree("tmp")
+
+
+@pytest.fixture
+def tmp_stac_collection():
+    original_collection_path = os.path.join(
+        EXAMPLE_DATA_DIR, "jaca_dataset_stac", "collection.json"
+    )
+
+    os.makedirs("tmp", exist_ok=True)
+    tmp_collection_path = os.path.join("tmp", "copied_collection")
+    shutil.copy(original_collection_path, tmp_collection_path)
+
+    copied_collection = Catalog.from_file(
+        os.path.join(tmp_collection_path, "collection.json")
+    )
+    yield copied_collection
+
+    shutil.rmtree("tmp")
+
+
+@pytest.fixture
+def sentinel_2():
+    os.makedirs("tmp", exist_ok=True)
+
+    yield os.path.join(EXAMPLE_DATA_DIR, 'sentinel_2')
+
+    shutil.rmtree("tmp")
+
+
+@pytest.fixture
+def tmp_sentinel_2():
+    original_dataset_path = os.path.join(EXAMPLE_DATA_DIR, "sentinel_2")
+
+    os.makedirs("tmp", exist_ok=True)
+    shutil.copytree(original_dataset_path, "tmp/sentinel_2")
+
+    yield "tmp/sentinel_2"
+
+    shutil.rmtree("tmp")

eotdl-2025.2.10/tests/unit/curation/stac/test_data/jaca_dataset_stac/catalog.json

@@ -0,0 +1,18 @@
+{
+  "type": "Catalog",
+  "id": "jaca-dataset-extensions",
+  "stac_version": "1.0.0",
+  "description": "Jaca dataset with STAC extensions",
+  "links": [
+    {
+      "rel": "root",
+      "href": "./catalog.json",
+      "type": "application/json"
+    },
+    {
+      "rel": "child",
+      "href": "./sentinel-2-l2a/collection.json",
+      "type": "application/json"
+    }
+  ]
+}