eotdl 2024.10.7__py3-none-any.whl → 2025.3.25__py3-none-any.whl

eotdl/curation/stac/extensions/raster.py

@@ -1,53 +0,0 @@
-"""
-Module for raster STAC extensions object
-"""
-
-from typing import Union, Optional
-
-import pystac
-import rasterio
-import pandas as pd
-
-from pystac.extensions.raster import RasterExtension, RasterBand
-from .base import STACExtensionObject
-
-
-class RasterExtensionObject(STACExtensionObject):
-    """
-    Raster extension object
-    """
-    def __init__(self) -> None:
-        super().__init__()
-
-    def add_extension_to_object(
-        self,
-        obj: Union[pystac.Item, pystac.Asset],
-        obj_info: Optional[pd.DataFrame] = None,
-    ) -> Union[pystac.Item, pystac.Asset]:
-        """
-        Add the extension to the given object
-
-        :param obj: object to add the extension
-        :param obj_info: object info from the STACDataFrame
-        """
-        if not isinstance(obj, pystac.Asset):
-            return obj
-        else:
-            raster_ext = RasterExtension.ext(obj, add_if_missing=True)
-            src = rasterio.open(obj.href)
-            bands = []
-            for band in src.indexes:
-                bands.append(
-                    RasterBand.create(
-                        nodata=src.nodatavals[band - 1],
-                        data_type=src.dtypes[band - 1],
-                        spatial_resolution=src.res,
-                    )
-                    if src.nodatavals
-                    else RasterBand.create(
-                        data_type=src.dtypes[band - 1], spatial_resolution=src.res
-                    )
-                )
-            raster_ext.apply(bands=bands)
-
-        return obj

eotdl/curation/stac/extensions/sar.py

@@ -1,55 +0,0 @@
-"""
-Module for SAR STAC extensions object
-"""
-
-from typing import Optional, Union
-
-import pystac
-import pandas as pd
-
-from pystac.extensions.sar import SarExtension
-from pystac.extensions.sar import FrequencyBand, Polarization
-
-from .base import STACExtensionObject
-
-
-class SarExtensionObject(STACExtensionObject):
-    """
-    SAR extension object
-    """
-    def __init__(self) -> None:
-        super().__init__()
-        self.polarizations = [Polarization.VV, Polarization.VH]
-        self.polarizations_dict = {"VV": Polarization.VV, "VH": Polarization.VH}
-
-    def add_extension_to_object(
-        self,
-        obj: Union[pystac.Item, pystac.Asset],
-        obj_info: Optional[pd.DataFrame] = None,
-    ) -> Union[pystac.Item, pystac.Asset]:
-        """
-        Add the extension to the given object
-
-        :param obj: object to add the extension
-        :param obj_info: object info from the STACDataFrame
-        """
-        # Add SAR extension to the item
-        sar_ext = SarExtension.ext(obj, add_if_missing=True)
-        if isinstance(obj, pystac.Item) or (
-            isinstance(obj, pystac.Asset)
-            and obj.title not in self.polarizations_dict.keys()
-        ):
-            polarizations = self.polarizations
-        elif (
-            isinstance(obj, pystac.Asset)
-            and obj.title in self.polarizations_dict.keys()
-        ):
-            polarizations = [self.polarizations_dict[obj.title]]
-        sar_ext.apply(
-            instrument_mode="EW",
-            polarizations=polarizations,
-            frequency_band=FrequencyBand.C,
-            product_type="GRD",
-        )
-
-        return obj

eotdl/curation/stac/extent.py

@@ -1,158 +0,0 @@
-"""
-Module for STAC extent
-"""
-
-from datetime import datetime
-from typing import List
-
-import pystac
-
-import rasterio
-
-from ...tools import get_item_metadata
-
-
-def get_dem_temporal_interval() -> pystac.TemporalExtent:
-    """
-    Get a temporal interval for DEM data
-    """
-    min_date = datetime.strptime("2011-01-01", "%Y-%m-%d")
-    max_date = datetime.strptime("2015-01-07", "%Y-%m-%d")
-
-    return pystac.TemporalExtent([(min_date, max_date)])
-
-
-def get_unknow_temporal_interval() -> pystac.TemporalExtent:
-    """
-    Get an unknown temporal interval
-    """
-    min_date = datetime.strptime("2000-01-01", "%Y-%m-%d")
-    max_date = datetime.strptime("2023-12-31", "%Y-%m-%d")
-
-    return pystac.TemporalExtent([(min_date, max_date)])
-
-
-def get_unknow_extent() -> pystac.Extent:
-    """
-    Get an unknown extent
-    """
-    return pystac.Extent(
-        spatial=pystac.SpatialExtent([[0, 0, 0, 0]]),
-        temporal=pystac.TemporalExtent(
-            [
-                (
-                    datetime.strptime("2000-01-01", "%Y-%m-%d"),
-                    datetime.strptime("2023-12-31", "%Y-%m-%d"),
-                )
-            ]
-        ),
-    )
-
-
-def get_collection_extent(rasters: List[str]) -> pystac.Extent:
-    """
-    Get the extent of a collection
-
-    :param rasters: list of rasters
-    """
-    # Get the spatial extent of the collection
-    spatial_extent = get_collection_spatial_extent(rasters)
-    # Get the temporal interval of the collection
-    temporal_interval = get_collection_temporal_interval(rasters)
-    # Create the Extent object
-    extent = pystac.Extent(spatial=spatial_extent, temporal=temporal_interval)
-
-    return extent
-
-
-def get_collection_spatial_extent(rasters: List[str]) -> pystac.SpatialExtent:
-    """
-    Get the spatial extent of a collection
-
-    :param path: path to the directory
-    """
-    # Get the bounding boxes of all the given rasters
-    bboxes = []
-    for raster in rasters:
-        with rasterio.open(raster) as ds:
-            bounds = ds.bounds
-            dst_crs = "EPSG:4326"
-            try:
-                left, bottom, right, top = rasterio.warp.transform_bounds(
-                    ds.crs, dst_crs, *bounds
-                )
-                bbox = [left, bottom, right, top]
-            except rasterio.errors.CRSError:
-                spatial_extent = pystac.SpatialExtent([[0, 0, 0, 0]])
-                return spatial_extent
-            bboxes.append(bbox)
-    # Get the minimum and maximum values of the bounding boxes
-    try:
-        left = min(bbox[0] for bbox in bboxes)
-        bottom = min(bbox[1] for bbox in bboxes)
-        right = max(bbox[2] for bbox in bboxes)
-        top = max(bbox[3] for bbox in bboxes)
-        spatial_extent = pystac.SpatialExtent([[left, bottom, right, top]])
-    except ValueError:
-        spatial_extent = pystac.SpatialExtent([[0, 0, 0, 0]])
-    finally:
-        return spatial_extent
-
-
-def get_collection_temporal_interval(rasters: List[str]) -> pystac.TemporalExtent:
-    """
-    Get the temporal interval of a collection
-
-    :param path: path to the directory
-    """
-    # Get all the metadata.json files in the directory of all the given rasters
-    metadata_jsons = []
-    for raster in rasters:
-        metadata_json = get_item_metadata(raster)
-        if metadata_json:
-            metadata_jsons.append(metadata_json)
-
-    if not metadata_jsons:
-        return (
-            get_unknow_temporal_interval()
-        )  # If there is no metadata, set a generic temporal interval
-
-    # Get the temporal interval of every metadata.json file and the type of the data
-    data_types = []
-    temporal_intervals = []
-    for metadata in metadata_jsons:
-        # Append the temporal interval to the list as a datetime object
-        temporal_intervals.append(metadata["acquisition-date"]) if metadata[
-            "acquisition-date"
-        ] else None
-        # Append the data type to the list
-        data_types.append(metadata["type"]) if metadata["type"] else None
-
-    if temporal_intervals:
-        min_date, max_date = None, None
-        try:
-            # Get the minimum and maximum values of the temporal intervals
-            min_date = min(
-                datetime.strptime(interval, "%Y-%m-%d")
-                for interval in temporal_intervals
-            )
-            max_date = max(
-                datetime.strptime(interval, "%Y-%m-%d")
-                for interval in temporal_intervals
-            )
-        except ValueError:
-            min_date = datetime.strptime("2000-01-01", "%Y-%m-%d")
-            max_date = datetime.strptime("2023-12-31", "%Y-%m-%d")
-        finally:
-            # Create the temporal interval
-            return pystac.TemporalExtent([(min_date, max_date)])
-    else:
-        # Check if the collection is composed by DEM data. If not, set a generic temporal interval
-        if (
-            set(data_types) == {"dem"}
-            or set(data_types) == {"DEM"}
-            or set(data_types) == {"dem", "DEM"}
-        ):
-            return get_dem_temporal_interval()
-        else:
-            return get_unknow_temporal_interval()

eotdl/curation/stac/parsers.py

@@ -1,61 +0,0 @@
-"""
-Module for STAC parsers
-"""
-
-from os.path import dirname, basename
-
-
-class STACIdParser:
-    """
-    STAC ID parser base class
-    """
-    def get_item_id(self, raster_path: str):
-        """
-        Get the ID of the STAC Item from the given raster path
-
-        :param raster_path: path to the raster file
-        """
-        return
-
-
-class StructuredParser(STACIdParser):
-    """
-    Structured STAC ID parser
-    """
-
-    def __init__(self) -> None:
-        super().__init__()
-
-    def get_item_id(self, raster_path: str):
-        """
-        Get the ID of the STAC Item from the given raster path.
-        This function assumes that the project given by the user is structured,
-        meaning that the raster files are stored in a folder with the same name
-
-        :param raster_path: path to the raster file
-        """
-        tiff_dir_path = dirname(raster_path)
-        item_id = tiff_dir_path.split("/")[-1]
-
-        return item_id
-
-
-class UnestructuredParser(STACIdParser):
-    """
-    Unstructured STAC ID parser
-    """
-
-    def __init__(self) -> None:
-        super().__init__()
-
-    def get_item_id(self, raster_path: str):
-        """
-        Get the ID of the STAC Item from the given raster path.
-        This function assumes that the project given by the user is unstructured,
-        meaning that the raster files are stored in the root folder or in a folder
-
-        :param raster_path: path to the raster file
-        """
-        item_id = basename(raster_path).split(".")[0]
-
-        return item_id

eotdl/datasets/download.py

@@ -1,104 +0,0 @@
-import os
-from pathlib import Path
-from tqdm import tqdm
-
-from ..auth import with_auth
-from .retrieve import retrieve_dataset, retrieve_dataset_files
-from ..repos import FilesAPIRepo, DatasetsAPIRepo
-from ..curation.stac import STACDataFrame
-from .metadata import generate_metadata
-
-
-@with_auth
-def download_dataset(
-    dataset_name,
-    version=None,
-    path=None,
-    logger=print,
-    assets=False,
-    force=False,
-    verbose=False,
-    user=None,
-    file=None,
-):
-    dataset = retrieve_dataset(dataset_name)
-    if version is None:
-        version = sorted(dataset["versions"], key=lambda v: v["version_id"])[-1][
-            "version_id"
-        ]
-    else:
-        assert version in [
-            v["version_id"] for v in dataset["versions"]
-        ], f"Version {version} not found"
-    download_base_path = os.getenv(
-        "EOTDL_DOWNLOAD_PATH", str(Path.home()) + "/.cache/eotdl/datasets"
-    )
-    if path is None:
-        download_path = download_base_path + "/" + dataset_name + "/v" + str(version)
-    else:
-        download_path = path + "/" + dataset_name + "/v" + str(version)
-    # check if dataset already exists
-    if os.path.exists(download_path) and not force:
-        os.makedirs(download_path, exist_ok=True)
-        raise Exception(
-            f"Dataset `{dataset['name']} v{str(version)}` already exists at {download_path}. To force download, use force=True or -f in the CLI."
-        )
-    if dataset["quality"] == 0:
-        if file:
-            raise NotImplementedError("Downloading a specific file is not implemented")
-        dataset_files = retrieve_dataset_files(dataset["id"], version)
-        repo = FilesAPIRepo()
-        for file in tqdm(dataset_files, disable=verbose, unit="file", position=0):
-            filename, file_version = file["filename"], file["version"]
-            if verbose:
-                logger(f"Downloading {file['filename']}...")
-            dst_path = repo.download_file(
-                dataset["id"],
-                filename,
-                user,
-                download_path,
-                file_version,
-                progress=True,
-            )
-            if verbose:
-                logger("Generating README.md ...")
-            generate_metadata(download_path, dataset)
-    else:
-        if verbose:
-            logger("Downloading STAC metadata...")
-        repo = DatasetsAPIRepo()
-        gdf, error = repo.download_stac(
-            dataset["id"],
-            user,
-        )
-        if error:
-            raise Exception(error)
-        df = STACDataFrame(gdf)
-        # df.geometry = df.geometry.apply(lambda x: Polygon() if x is None else x)
-        df.to_stac(download_path)
-        # download assets
-        if assets:
-            if verbose:
-                logger("Downloading assets...")
-            repo = FilesAPIRepo()
-            df = df.dropna(subset=["assets"])
-            for row in tqdm(df.iterrows(), total=len(df)):
-                for k, v in row[1]["assets"].items():
-                    href = v["href"]
-                    _, filename = href.split("/download/")
-                    # will overwrite assets with same name :(
-                    repo.download_file_url(
-                        href, filename, f"{download_path}/assets", user
-                    )
-        else:
-            logger("To download assets, set assets=True or -a in the CLI.")
-    if verbose:
-        logger("Done")
-    return download_path
-
-
-@with_auth
-def download_file_url(url, path, progress=True, logger=print, user=None):
-    repo = FilesAPIRepo()
-    _, filename = url.split("/download/")
-    return repo.download_file_url(url, filename, f"{path}/assets", user, progress)

eotdl/files/list_files.py

@@ -1,13 +0,0 @@
-from ..datasets import retrieve_dataset, retrieve_dataset_files
-from ..models import retrieve_model, retrieve_model_files
-
-def list_files(dataset_or_model_name, version=1):
-	try:
-		dataset = retrieve_dataset(dataset_or_model_name)
-		return retrieve_dataset_files(dataset['id'], version)
-	except Exception as e:
-		try:
-			model = retrieve_model(dataset_or_model_name)
-			return retrieve_model_files(model['id'], version)
-		except Exception as e:
-			raise Exception(f"Dataset or model {dataset_or_model_name} not found.")

eotdl/models/download.py

@@ -1,101 +0,0 @@
-import os
-from pathlib import Path
-from tqdm import tqdm
-
-from ..auth import with_auth
-from .retrieve import retrieve_model, retrieve_model_files
-from ..shared import calculate_checksum
-from ..repos import FilesAPIRepo, ModelsAPIRepo
-from .metadata import generate_metadata
-from ..curation.stac import STACDataFrame
-
-
-@with_auth
-def download_model(
-    model_name,
-    version=None,
-    path=None,
-    logger=None,
-    assets=False,
-    force=False,
-    verbose=False,
-    user=None,
-    file=None,
-):
-    model = retrieve_model(model_name)
-    if version is None:
-        version = sorted(model["versions"], key=lambda v: v["version_id"])[-1][
-            "version_id"
-        ]
-    else:
-        assert version in [
-            v["version_id"] for v in model["versions"]
-        ], f"Version {version} not found"
-    download_base_path = os.getenv(
-        "EOTDL_DOWNLOAD_PATH", str(Path.home()) + "/.cache/eotdl/models"
-    )
-    if path is None:
-        download_path = download_base_path + "/" + model_name + "/v" + str(version)
-    else:
-        download_path = path + "/" + model_name + "/v" + str(version)
-    # check if model already exists
-    if os.path.exists(download_path) and not force:
-        os.makedirs(download_path, exist_ok=True)
-        raise Exception(
-            f"model `{model['name']} v{str(version)}` already exists at {download_path}. To force download, use force=True or -f in the CLI."
-        )
-    if model["quality"] == 0:
-        if file:
-            raise NotImplementedError("Downloading a specific file is not implemented")
-        model_files = retrieve_model_files(model["id"], version)
-        repo = FilesAPIRepo()
-        for file in tqdm(model_files, disable=verbose, unit="file"):
-            filename, file_version = file["filename"], file["version"]
-            if verbose:
-                logger(f"Downloading {file['filename']}...")
-            dst_path = repo.download_file(
-                model["id"],
-                filename,
-                user,
-                download_path,
-                file_version,
-                endpoint="models",
-            )
-            if verbose:
-                logger("Generating README.md ...")
-            generate_metadata(download_path, model)
-    else:
-        if verbose:
-            logger("Downloading STAC metadata...")
-        repo = ModelsAPIRepo()
-        gdf, error = repo.download_stac(
-            model["id"],
-            user,
-        )
-        if error:
-            raise Exception(error)
-        print(gdf)
-        df = STACDataFrame(gdf)
-        # df.geometry = df.geometry.apply(lambda x: Polygon() if x is None else x)
-        df.to_stac(download_path)
-        print("----")
-        print(df)
-        # download assets
-        if assets:
-            if verbose:
-                logger("Downloading assets...")
-            repo = FilesAPIRepo()
-            df = df.dropna(subset=["assets"])
-            for row in tqdm(df.iterrows(), total=len(df)):
-                for k, v in row[1]["assets"].items():
-                    href = v["href"]
-                    _, filename = href.split("/download/")
-                    # will overwrite assets with same name :(
-                    repo.download_file_url(
-                        href, filename, f"{download_path}/assets", user
-                    )
-        else:
-            logger("To download assets, set assets=True or -a in the CLI.")
-    if verbose:
-        logger("Done")
-    return download_path

eotdl/models/metadata.py

@@ -1,43 +0,0 @@
-from pydantic import BaseModel, validator
-from typing import List, Optional
-from pathlib import Path
-
-
-class Metadata(BaseModel):
-    authors: List[str]
-    license: str
-    source: str
-    name: str
-    thumbnail: Optional[str] = ""
-
-    # validate source is a URL
-    @validator("source")
-    def source_is_url(cls, v):
-        if not v.startswith("http") and not v.startswith("https"):
-            raise ValueError("source must be a URL")
-        return v
-
-    # validate thumbnail is a url
-    @validator("thumbnail")
-    def thumbnail_is_url(cls, v):
-        if not v.startswith("http") and not v.startswith("https"):
-            raise ValueError("thumbnail must be a URL")
-        return v
-
-
-def generate_metadata(download_path, model):
-    with open(download_path + "/README.md", "w") as f:
-        f.write("---\n")
-        f.write(f"name: {model['name']}\n")
-        f.write(f"license: {model['license']}\n")
-        f.write(f"source: {model['source']}\n")
-        f.write(f"thumbnail: {model['thumbnail']}\n")
-        f.write(f"authors:\n")
-        for author in model["authors"]:
-            f.write(f"  - {author}\n")
-        f.write("---\n")
-        f.write(model["description"])
-    # remove metadata.yml if exists
-    if Path(download_path + "/metadata.yml").exists():
-        Path(download_path + "/metadata.yml").unlink()
-    return download_path + "/README.md"

eotdl/wrappers/utils.py

@@ -1,35 +0,0 @@
-from ..curation.stac import STACDataFrame
-
-def download_model(model_name, dst_path, version, force=False, download=True):
-    # check if model already downloaded
-    version = 1 if version is None else version
-    download_path = dst_path + "/" + model_name + "/v" + str(version)
-    if os.path.exists(download_path) and not force:
-        df = STACDataFrame.from_stac_file(download_path + f"/{model_name}/catalog.json")
-        return download_path, df
-    # check model exists
-    model, error = retrieve_model(model_name)
-    if error:
-        raise Exception(error)
-    if model["quality"] < 2:
-        raise Exception("Only Q2+ models are supported")
-    # check version exist
-    assert version in [
-        v["version_id"] for v in model["versions"]
-    ], f"Version {version} not found"
-    # download model files
-    gdf, error = retrieve_model_stac(model["id"], version)
-    if error:
-        raise Exception(error)
-    df = STACDataFrame(gdf)
-    if not download:
-        return download_path, df
-    os.makedirs(download_path, exist_ok=True)
-    df.to_stac(download_path)
-    df = df.dropna(subset=["assets"])
-    for row in df.iterrows():
-        for k, v in row[1]["assets"].items():
-            href = v["href"]
-            _, filename = href.split("/download/")
-            download_file_url(href, filename, f"{download_path}/assets")
-    return download_path, df