eotdl 2023.10.25.post10__py3-none-any.whl → 2023.11.2.post2__py3-none-any.whl

eotdl/repos/FilesAPIRepo.py

@@ -1,6 +1,7 @@
 import requests
 import os
 from tqdm import tqdm
+import hashlib
 
 from ..repos import APIRepo
 
@@ -9,52 +10,85 @@ class FilesAPIRepo(APIRepo):
     def __init__(self, url=None):
         super().__init__(url)
 
-    def ingest_file(self, file, dataset_id, version, parent, id_token, checksum=None):
+    def ingest_files_batch(
+        self,
+        batch,  # ziped batch of files
+        checksums,
+        dataset_or_model_id,
+        id_token,
+        endpoint,
+        version=None,
+    ):
+        url = self.url + f"{endpoint}/{dataset_or_model_id}"
+        if version is not None:
+            url += "?version=" + str(version)
         reponse = requests.post(
-            self.url + "datasets/" + dataset_id,
-            files={"file": open(file, "rb")},
-            data={"checksum": checksum, "version": version, "parent": parent}
-            if checksum
-            else None,
+            url,
+            files={"batch": ("batch.zip", batch)},
+            data={"checksums": checksums},
             headers={"Authorization": "Bearer " + id_token},
         )
         return self.format_response(reponse)
 
-    def ingest_existing_file(
+    def add_files_batch_to_version(
         self,
-        filename,
-        dataset_id,
+        batch,
+        dataset_or_model_id,
         version,
-        file_version,
         id_token,
-        checksum=None,
+        endpoint,
     ):
         reponse = requests.post(
-            self.url + "datasets/" + dataset_id,
+            self.url + f"{endpoint}/{dataset_or_model_id}/files?version={str(version)}",
             data={
-                "checksum": checksum,
-                "version": version,
-                "filename": filename,
-                "fileversion": file_version,
-            }
+                "filenames": [f["path"] for f in batch],
+                "checksums": [f["checksum"] for f in batch],
+            },
+            headers={"Authorization": "Bearer " + id_token},
+        )
+        return self.format_response(reponse)
+
+    def ingest_file(
+        self,
+        file,
+        dataset_or_model_id,
+        version,
+        parent,
+        id_token,
+        checksum,
+        endpoint,
+    ):
+        reponse = requests.post(
+            self.url + f"{endpoint}/{dataset_or_model_id}",
+            files={"file": open(file, "rb")},
+            data={"checksum": checksum, "version": version, "parent": parent}
             if checksum
             else None,
             headers={"Authorization": "Bearer " + id_token},
         )
         return self.format_response(reponse)
 
-    def retrieve_dataset_files(self, dataset_id, version=None):
-        url = self.url + "datasets/" + dataset_id + "/files"
+    def retrieve_files(self, dataset_or_model_id, endpoint, version=None):
+        url = f"{self.url}{endpoint}/{dataset_or_model_id}/files"
         if version is not None:
             url += "?version=" + str(version)
         response = requests.get(url)
         return self.format_response(response)
 
-    def download_file(self, dataset_id, file_name, id_token, path, file_version):
-        url = self.url + "datasets/" + dataset_id + "/download/" + file_name
+    def download_file(
+        self,
+        dataset_or_model_id,
+        file_name,
+        id_token,
+        path,
+        file_version,
+        endpoint="datasets",
+        progress=False,
+    ):
+        url = self.url + f"{endpoint}/{dataset_or_model_id}/download/{file_name}"
         if file_version is not None:
             url += "?version=" + str(file_version)
-        return self.download_file_url(url, file_name, path, id_token)
+        return self.download_file_url(url, file_name, path, id_token, progress=progress)
 
     def download_file_url(self, url, filename, path, id_token, progress=False):
         headers = {"Authorization": "Bearer " + id_token}
@@ -66,9 +100,14 @@ class FilesAPIRepo(APIRepo):
             r.raise_for_status()
             total_size = int(r.headers.get("content-length", 0))
             block_size = 1024 * 1024 * 10
+            progress = progress and total_size > 1024 * 1024 * 16
             if progress:
                 progress_bar = tqdm(
-                    total=total_size, unit="iB", unit_scale=True, unit_divisor=1024
+                    total=total_size,
+                    unit="iB",
+                    unit_scale=True,
+                    unit_divisor=1024,
+                    position=1,
                 )
             with open(path, "wb") as f:
                 for chunk in r.iter_content(block_size):
@@ -90,79 +129,81 @@ class FilesAPIRepo(APIRepo):
     #         return None, reponse.json()["detail"]
     #     return reponse.json(), None
 
-    # def read_in_chunks(self, file_object, CHUNK_SIZE):
-    #     while True:
-    #         data = file_object.read(CHUNK_SIZE)
-    #         if not data:
-    #             break
-    #         yield data
-
-    # def prepare_large_upload(self, file, dataset_id, checksum, id_token):
-    #     filename = Path(file).name
-    #     response = requests.post(
-    #         self.url + f"datasets/{dataset_id}/uploadId",
-    #         json={"name": filename, "checksum": checksum},
-    #         headers={"Authorization": "Bearer " + id_token},
-    #     )
-    #     if response.status_code != 200:
-    #         raise Exception(response.json()["detail"])
-    #     data = response.json()
-    #     upload_id, parts = (
-    #         data["upload_id"],
-    #         data["parts"] if "parts" in data else [],
-    #     )
-    #     return upload_id, parts
-
-    # def get_chunk_size(self, content_size):
-    #     # adapt chunk size to content size to avoid S3 limits (10000 parts, 500MB per part, 5TB per object)
-    #     chunk_size = 1024 * 1024 * 10  # 10 MB (up to 100 GB, 10000 parts)
-    #     if content_size >= 1024 * 1024 * 1024 * 100:  # 100 GB
-    #         chunk_size = 1024 * 1024 * 100  # 100 MB (up to 1 TB, 10000 parts)
-    #     elif content_size >= 1024 * 1024 * 1024 * 1000:  # 1 TB
-    #         chunk_size = 1024 * 1024 * 500  # 0.5 GB (up to 5 TB, 10000 parts)
-    #     return chunk_size
-
-    # def ingest_large_dataset(self, file, upload_id, id_token, parts):
-    #     content_path = os.path.abspath(file)
-    #     content_size = os.stat(content_path).st_size
-    #     chunk_size = self.get_chunk_size(content_size)
-    #     total_chunks = content_size // chunk_size
-    #     # upload chunks sequentially
-    #     pbar = tqdm(
-    #         self.read_in_chunks(open(content_path, "rb"), chunk_size),
-    #         total=total_chunks,
-    #     )
-    #     index = 0
-    #     for chunk in pbar:
-    #         part = index // chunk_size + 1
-    #         offset = index + len(chunk)
-    #         index = offset
-    #         if part not in parts:
-    #             checksum = hashlib.md5(chunk).hexdigest()
-    #             response = requests.post(
-    #                 self.url + "datasets/chunk/" + upload_id,
-    #                 files={"file": chunk},
-    #                 data={"part_number": part, "checksum": checksum},
-    #                 headers={"Authorization": "Bearer " + id_token},
-    #             )
-    #             if response.status_code != 200:
-    #                 raise Exception(response.json()["detail"])
-    #         pbar.set_description(
-    #             "{:.2f}/{:.2f} MB".format(
-    #                 offset / 1024 / 1024, content_size / 1024 / 1024
-    #             )
-    #         )
-    #     pbar.close()
-    #     return
+    def prepare_large_upload(
+        self, filename, dataset_or_model_id, checksum, id_token, endpoint
+    ):
+        response = requests.post(
+            self.url + f"{endpoint}/{dataset_or_model_id}/uploadId",
+            json={"filname": filename, "checksum": checksum},
+            headers={"Authorization": "Bearer " + id_token},
+        )
+        if response.status_code != 200:
+            raise Exception(response.json()["detail"])
+        data = response.json()
+        upload_id, parts = (
+            data["upload_id"],
+            data["parts"] if "parts" in data else [],
+        )
+        return upload_id, parts
 
-    # def complete_upload(self, id_token, upload_id):
-    #     r = requests.post(
-    #         self.url + "datasets/complete/" + upload_id,
-    #         headers={"Authorization": "Bearer " + id_token},
-    #     )
-    #     if r.status_code != 200:
-    #         return None, r.json()["detail"]
-    #     return r.json(), None
+    def get_chunk_size(self, content_size):
+        # adapt chunk size to content size to avoid S3 limits (10000 parts, 500MB per part, 5TB per object)
+        chunk_size = 1024 * 1024 * 10  # 10 MB (up to 100 GB, 10000 parts)
+        if content_size >= 1024 * 1024 * 1024 * 100:  # 100 GB
+            chunk_size = 1024 * 1024 * 100  # 100 MB (up to 1 TB, 10000 parts)
+        elif content_size >= 1024 * 1024 * 1024 * 1000:  # 1 TB
+            chunk_size = 1024 * 1024 * 500  # 0.5 GB (up to 5 TB, 10000 parts)
+        return chunk_size
+
+    def read_in_chunks(self, file_object, CHUNK_SIZE):
+        while True:
+            data = file_object.read(CHUNK_SIZE)
+            if not data:
+                break
+            yield data
+
+    def ingest_large_file(
+        self, file_path, files_size, upload_id, id_token, parts, endpoint
+    ):
+        print(endpoint)
+        # content_path = os.path.abspath(file)
+        # content_size = os.stat(content_path).st_size
+        chunk_size = self.get_chunk_size(files_size)
+        total_chunks = files_size // chunk_size
+        # upload chunks sequentially
+        pbar = tqdm(
+            self.read_in_chunks(open(file_path, "rb"), chunk_size),
+            total=total_chunks,
+        )
+        index = 0
+        for chunk in pbar:
+            part = index // chunk_size + 1
+            offset = index + len(chunk)
+            index = offset
+            if part not in parts:
+                checksum = hashlib.md5(chunk).hexdigest()
+                response = requests.post(
+                    f"{self.url}{endpoint}/chunk/{upload_id}",
+                    files={"file": chunk},
+                    data={"part_number": part, "checksum": checksum},
+                    headers={"Authorization": "Bearer " + id_token},
+                )
+                if response.status_code != 200:
+                    raise Exception(response.json()["detail"])
+            pbar.set_description(
+                "{:.2f}/{:.2f} MB".format(
+                    offset / 1024 / 1024, files_size / 1024 / 1024
+                )
+            )
+        pbar.close()
+        return
+
+    def complete_upload(self, id_token, upload_id, version, endpoint):
+        r = requests.post(
+            f"{self.url}{endpoint}/complete/{upload_id}?version={version}",
+            headers={"Authorization": "Bearer " + id_token},
+        )
+        return self.format_response(r)
 
     # def update_dataset(self, name, path, id_token, checksum):
     #     # check that dataset exists

eotdl/repos/ModelsAPIRepo.py

@@ -0,0 +1,40 @@
+import requests
+import os
+
+from ..repos import APIRepo
+
+
+class ModelsAPIRepo(APIRepo):
+    def __init__(self, url=None):
+        super().__init__(url)
+
+    def retrieve_models(self, name, limit):
+        url = self.url + "models"
+        if name is not None:
+            url += "?match=" + name
+        if limit is not None:
+            if name is None:
+                url += "?limit=" + str(limit)
+            else:
+                url += "&limit=" + str(limit)
+        response = requests.get(url)
+        return self.format_response(response)
+
+    def create_model(self, metadata, id_token):
+        response = requests.post(
+            self.url + "models",
+            json=metadata,
+            headers={"Authorization": "Bearer " + id_token},
+        )
+        return self.format_response(response)
+
+    def retrieve_model(self, name):
+        response = requests.get(self.url + "models?name=" + name)
+        return self.format_response(response)
+
+    def create_version(self, model_id, id_token):
+        response = requests.post(
+            self.url + "models/version/" + model_id,
+            headers={"Authorization": "Bearer " + id_token},
+        )
+        return self.format_response(response)

eotdl/repos/__init__.py

@@ -3,3 +3,4 @@ from .APIRepo import APIRepo
 from .AuthAPIRepo import AuthAPIRepo
 from .DatasetsAPIRepo import DatasetsAPIRepo
 from .FilesAPIRepo import FilesAPIRepo
+from .ModelsAPIRepo import ModelsAPIRepo

eotdl/shared/__init__.py

@@ -0,0 +1 @@
+from .checksum import calculate_checksum

eotdl/tools/__init__.py

@@ -1,7 +1,6 @@
-from .stac import stac_items_to_gdf
-from .tools import (generate_location_payload,
-                    format_product_location_payload,
-                    get_images_by_location,
-                    get_tarfile_image_info)
+from .stac import *
+from .tools import *
 from .geo_utils import *
-from .time_utils import *
+from .time_utils import *
+from .metadata import *
+from .paths import *

eotdl/tools/geo_utils.py

@@ -5,9 +5,10 @@ import tarfile
 
 from typing import Union
 from shapely import geometry
-from shapely.geometry import box, Polygon
+from shapely.geometry import box, Polygon, shape
 from pyproj import Transformer
 from sentinelhub import BBox, CRS, bbox_to_dimensions
+from pandas import isna
 
 
 def is_bounding_box(bbox: list) -> bool:
@@ -199,3 +200,16 @@ def generate_new_locations_bounding_boxes(gdf: gpd.GeoDataFrame,
         latest_id += 1
 
     return bbox_by_new_location
+
+
+def convert_df_geom_to_shape(row):
+    """
+    Convert the geometry of a dataframe row to a shapely shape
+    """
+    if not isna(row["geometry"]):
+        geo = shape(row["geometry"])
+        wkt = geo.wkt
+    else:
+        wkt = "POLYGON EMPTY"
+
+    return wkt

eotdl/tools/stac.py

@@ -1,11 +1,20 @@
 """
 Module for data engineering with STAC elements
 """
+
 import geopandas as gpd
-from pystac import ItemCollection
+import pystac
+
+from os.path import dirname, join, abspath
+from os import makedirs
+from json import dumps
+from typing import Union, Optional
+from tqdm import tqdm
+from traceback import print_exc
+from shutil import rmtree
 
 
-def stac_items_to_gdf(items: ItemCollection) -> gpd.GeoDataFrame:
+def stac_items_to_gdf(items: pystac.ItemCollection) -> gpd.GeoDataFrame:
     """
     Get a GeoDataFrame from a given pystac.ItemCollection. 
 
@@ -18,14 +27,141 @@ def stac_items_to_gdf(items: ItemCollection) -> gpd.GeoDataFrame:
     features = []
     for f in _features:
         if f not in features:
-            # Add the id, type and stac_extensions as properties, in order to retrieve 
-            # them as columns in the GeoDataFrame
-            # TODO put all the properties in the GeoDataFrame
-            f['properties']['id'] = f['id']
+            # Add all the keys in the properties dict as columns in the GeoDataFrame
+            for k, v in f.items():
+                if k not in f['properties'] and k != 'geometry':
+                    f['properties'][k] = v
             if 'scene_id' in f['properties']:
                 f['properties']['scene_id'] = f['id'].split('_')[3]
-            f['properties']['type'] = f['type']
-            f['properties']['stac_extensions'] = f['stac_extensions']
             features.append(f)       
     
     return gpd.GeoDataFrame.from_features(features)
+
+
+def get_all_children(obj: pystac.STACObject) -> list:
+    """
+    Get all the children of a STAC object
+    """
+    children = []
+    # Append the current object to the list
+    children.append(obj.to_dict())
+
+    # Collections
+    collections = list(obj.get_collections())
+    for collection in collections:
+        children.append(collection.to_dict())
+
+    # Items
+    items = obj.get_items()
+    for item in items:
+        children.append(item.to_dict())
+
+    # Items from collections
+    for collection in collections:
+        items = collection.get_items()
+        for item in items:
+            children.append(item.to_dict())
+
+    return children
+
+
+def make_links_relative_to_path(path: str,
+                                catalog: Union[pystac.Catalog, str],
+                                ) -> pystac.Catalog:
+    """
+    Makes all asset HREFs and links in the STAC catalog relative to a given path
+    """
+    if isinstance(catalog, str):
+        catalog = pystac.read_file(catalog)
+    path = abspath(path)
+
+    # Create a temporary catalog in the destination path to set as root
+    future_path = join(path, 'catalog.json')
+    makedirs(path, exist_ok=True)
+    with open(future_path, 'w') as f:
+        f.write(dumps(catalog.to_dict(), indent=4))
+    temp_catalog = pystac.Catalog.from_file(future_path)
+
+    catalog.set_root(temp_catalog)
+    catalog.make_all_asset_hrefs_absolute()
+
+    for collection in catalog.get_children():
+        # Create new collection
+        new_collection = collection.clone()
+        new_collection.set_self_href(join(path, collection.id, f"collection.json"))
+        new_collection.set_root(catalog)
+        new_collection.set_parent(catalog)
+        # Remove old collection and add new one to catalog
+        catalog.remove_child(collection.id)
+        catalog.add_child(new_collection)
+        for item in collection.get_all_items():
+            # Create new item from old collection and add it to the new collection
+            new_item = item.clone()
+            new_item.set_self_href(join(path, collection.id, item.id, f"{item.id}.json"))
+            new_item.set_parent(collection)
+            new_item.set_root(catalog)
+            new_item.make_asset_hrefs_relative()
+            new_collection.add_item(new_item)
+
+    catalog.make_all_asset_hrefs_relative()
+
+    return catalog
+
+
+def merge_stac_catalogs(catalog_1: Union[pystac.Catalog, str],
+                        catalog_2: Union[pystac.Catalog, str],
+                        destination: Optional[str] = None,
+                        keep_extensions: Optional[bool] = False,
+                        catalog_type: Optional[pystac.CatalogType] = pystac.CatalogType.SELF_CONTAINED
+                        ) -> None:
+    """
+    Merge two STAC catalogs, keeping the properties, collection and items of both catalogs
+    """
+    if isinstance(catalog_1, str):
+        catalog_1 = pystac.Catalog.from_file(catalog_1)
+    if isinstance(catalog_2, str):
+        catalog_2 = pystac.Catalog.from_file(catalog_2)
+
+    for col1 in tqdm(catalog_1.get_children(), desc='Merging catalogs...'):
+        # Check if the collection exists in catalog_2
+        col2 = catalog_2.get_child(col1.id)
+        if col2 is None:
+            # If it does not exist, add it
+            col1_ = col1.clone()
+            catalog_2.add_child(col1)
+            col2 = catalog_2.get_child(col1.id)
+            col2.clear_items()
+            for i in col1_.get_stac_objects(pystac.RelType.ITEM):
+                col2.add_item(i)
+        else:
+            # If it exists, merge the items
+            for item1 in col1.get_items():
+                if col2.get_item(item1.id) is None:
+                    col2.add_item(item1)
+
+    if keep_extensions:
+        for ext in catalog_1.stac_extensions:
+            if ext not in catalog_2.stac_extensions:
+                catalog_2.stac_extensions.append(ext)
+
+        for extra_field_name, extra_field_value in catalog_1.extra_fields.items():
+            if extra_field_name not in catalog_2.extra_fields:
+                catalog_2.extra_fields[extra_field_name] = extra_field_value
+
+    if destination:
+        make_links_relative_to_path(destination, catalog_2)
+    else:
+        destination = dirname(catalog_2.get_self_href())
+
+    # Save the merged catalog
+    try:
+        print("Validating and saving...")
+        catalog_2.validate()
+        rmtree(destination) if not destination else None # Remove the old catalog and replace it with the new one
+        catalog_2.normalize_and_save(root_href=destination, 
+                                     catalog_type=catalog_type
+                                     )
+        print("Success!")
+    except pystac.STACValidationError:
+        # Return full callback
+        print_exc()

eotdl/tools/time_utils.py

@@ -81,21 +81,21 @@ def expand_time_interval(time_interval: Union[list, tuple], format: str='%Y-%m-%
 def prepare_time_interval(date):
     if isinstance(date, str):
         date = datetime.strptime(date, "%Y-%m-%d")
-    elif isinstance(date, datetime):
-        date = date.strftime("%Y-%m-%d")
     elif isinstance(date, tuple):
         if not is_time_interval(date):
             raise ValueError('The time interval must be a range of two dates, with format YYYY-MM-DD or a datetime object')
         else:
             return date
-    else:
+    elif not isinstance(date, datetime):
         raise ValueError('The date must be a string with format YYYY-MM-DD or a datetime object')
+
     date_day_before = date - timedelta(days=1)
     date_next_day = date + timedelta(days=1)
-    date_day_before = date_day_before.strftime("%Y-%m-%d")
-    date_next_day = date_next_day.strftime("%Y-%m-%d")
 
-    return (date_day_before, date_next_day)
+    date_day_before_str = date_day_before.strftime("%Y-%m-%d")
+    date_next_day_str = date_next_day.strftime("%Y-%m-%d")
+
+    return (date_day_before_str, date_next_day_str)
 
 
 def get_day_between(from_date: Union[datetime, str], 
@@ -112,3 +112,16 @@ def get_day_between(from_date: Union[datetime, str],
     date_between = date_between.strftime("%Y-%m-%d") 
     
     return date_between
+
+
+def format_time_acquired(dt: Union[str, datetime]) -> str:
+    """
+    Format the date time to the required format for STAC
+
+    :param dt: date time to format
+    """
+    from dateutil import parser
+    
+    dt_str = parser.parse(dt).strftime("%Y-%m-%dT%H:%M:%S.%f")
+
+    return dt_str

eotdl/tools/tools.py

@@ -5,13 +5,12 @@ Module for data engineeringt
 import geopandas as gpd
 import pandas as pd
 import tarfile
-import rasterio
 import re
 import datetime
 import json
 
-from shapely.geometry import box, Polygon
-from pyproj import Transformer
+from .geo_utils import get_image_bbox
+from shapely.geometry import box
 from os.path import exists
 from typing import Union, Optional
 

{eotdl-2023.10.25.post10.dist-info → eotdl-2023.11.2.post2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: eotdl
-Version: 2023.10.25.post10
+Version: 2023.11.2.post2
 Summary: Earth Observation Training Data Lab
 License: MIT
 Author: EarthPulse