eotdl 2024.10.7__py3-none-any.whl → 2025.3.25__py3-none-any.whl

eotdl/curation/stac/stac.py

@@ -1,343 +1,33 @@
-"""
-Module for generating STAC metadata
-"""
-
-import random
-from datetime import datetime
-from typing import Union, Optional
-from os.path import join, basename, dirname
-
-import pandas as pd
-import pystac
-import rasterio
+import pyarrow.parquet as pq
+import stac_geoparquet
+import json
 from tqdm import tqdm
-from shapely.geometry import Polygon, mapping
-
-from .parsers import STACIdParser, StructuredParser
-from .assets import STACAssetGenerator
-from .dataframe_labeling import LabelingStrategy, UnlabeledStrategy
-from ...tools import (
-    format_time_acquired,
-    cut_images,
-    get_item_metadata,
-    get_all_images_in_path,
-)
-from .extensions import (
-    type_stac_extensions_dict,
-    SUPPORTED_EXTENSIONS,
-)
-from .extent import get_collection_extent
-
-
-class STACGenerator:
-    """
-    STAC generator class
-    """
-
-    def __init__(
-        self,
-        image_format: str = "tiff",
-        catalog_type: pystac.CatalogType = pystac.CatalogType.SELF_CONTAINED,
-        item_parser: STACIdParser = StructuredParser,
-        assets_generator: STACAssetGenerator = STACAssetGenerator,
-        labeling_strategy: LabelingStrategy = UnlabeledStrategy,
-    ) -> None:
-        """
-        Initialize the STAC generator
-
-        :param image_format: image format of the assets
-        :param catalog_type: type of the catalog
-        :param item_parser: parser to get the item ID
-        :param assets_generator: generator to generate the assets
-        :param labeling_strategy: strategy to label the images
-        """
-        self._image_format = image_format
-        self._catalog_type = catalog_type
-        self._item_parser = item_parser()
-        self._assets_generator = assets_generator()
-        self._labeling_strategy = labeling_strategy()
-        self._extensions_dict: dict = type_stac_extensions_dict
-        self._stac_dataframe = pd.DataFrame()
-
-    def generate_stac_metadata(
-        self,
-        stac_id: str,
-        description: str,
-        stac_dataframe: pd.DataFrame = None,
-        output_folder: str = "stac",
-        **kwargs,
-    ) -> None:
-        """
-        Generate STAC metadata for a given directory containing the assets to generate metadata
-
-        :param id: id of the catalog
-        :param description: description of the catalog
-        :param stac_dataframe: dataframe with the STAC metadata of a given directory containing the assets to generate metadata
-        :param output_folder: output folder to write the catalog to
-        """
-        self._stac_dataframe = (
-            stac_dataframe if self._stac_dataframe.empty else self._stac_dataframe
-        )
-        if self._stac_dataframe.empty:
-            raise ValueError("No STAC dataframe provided")
-
-        # Create an empty catalog
-        catalog = pystac.Catalog(id=stac_id, description=description, **kwargs)
-
-        # Add the collections to the catalog
-        collections = self._stac_dataframe.collection.unique()
-        for collection_path in collections:
-            # Generate the collection
-            collection = self.generate_stac_collection(collection_path)
-            # Add the collection to the catalog
-            catalog.add_child(collection)
-
-        # Check there have been generate all the items from the images
-        items_count = 0
-        for collection in catalog.get_children():
-            items = list(
-                set([item.id for item in collection.get_items(recursive=True)])
-            )
-            items_count += len(items)
-        if len(self._stac_dataframe) != items_count:
-            raise pystac.STACError(
-                "Not all the STAC items have been generated, please check the Item parser or the STAC dataframe. If you are using the StructuredParser, check that the images are in the correct folder structure."
-            )
-
-        # Add the catalog to the root directory
-        catalog.normalize_hrefs(output_folder)
-
-        # Validate the catalog
-        print("Validating and saving catalog...")
-        try:
-            pystac.validation.validate(catalog)
-            catalog.save(catalog_type=self._catalog_type)
-            print("Success!")
-        except pystac.STACValidationError as e:
-            print(f"Catalog validation error: {e}")
-            return
-
-    def get_stac_dataframe(
-        self,
-        path: str,
-        collections: Optional[Union[str, dict]] = "source",
-        bands: Optional[dict] = None,
-        extensions: Optional[dict] = None,
-        sample: Optional[int] = None,
-    ) -> pd.DataFrame:
-        """
-        Get a dataframe with the STAC metadata of a given directory containing the assets to generate metadata
-
-        :param path: path to the root directory
-        :param collections: dictionary with the collections
-        :param bands: dictionary with the bands
-        :param extensions: dictionary with the extensions
-        """
-        images = get_all_images_in_path(path, self._image_format)
-        if len(images) == 0:
-            raise ValueError(
-                "No images found in the given path with the given extension. Please check the path and the extension"
-            )
-
-        if self._assets_generator.type == "Extracted":
-            images = cut_images(images)
-
-        if sample:
-            try:
-                images = random.sample(images, sample)
-            except ValueError:
-                raise ValueError(
-                    f"Sample size must be smaller than the number of images ({len(images)}). May be there are no images found in the given path with the given extension"
-                )
-
-        labels, ixs = self._labeling_strategy.get_images_labels(images)
-        bands_values = self._get_items_list_from_dict(labels, bands)
-        extensions_values = self._get_items_list_from_dict(labels, extensions)
-
-        if collections == "source":
-            # List of path with the same value repeated as many times as the number of images
-            collections_values = [join(path, "source") for i in range(len(images))]
-        elif collections == "*":
-            collections_values = [
-                join(path, basename(dirname(image))) for image in images
-            ]
-        else:
-            try:
-                collections_values = [
-                    join(path, value)
-                    for value in self._get_items_list_from_dict(labels, collections)
-                ]
-            except TypeError:
-                raise pystac.STACError(
-                    "There is an error generating the collections. Please check the collections dictionary"
-                )
-
-        df = pd.DataFrame(
-            {
-                "image": images,
-                "label": labels,
-                "ix": ixs,
-                "collection": collections_values,
-                "extensions": extensions_values,
-                "bands": bands_values,
-            }
-        )
-
-        self._stac_dataframe = df
-
-        return df
-
-    def _get_items_list_from_dict(self, labels: list, items: dict) -> list:
-        """
-        Get a list of items from a dictionary
-
-        :param labels: list of labels
-        :param items: dictionary with the items
-        """
-        if not items:
-            # Create list of None with the same length as the labels list
-            return [None for _ in labels]
-        items_list = []
-        for label in labels:
-            if label in items.keys():
-                items_list.append(items[label])
-            else:
-                items_list.append(None)
-
-        return items_list
-
-    def generate_stac_collection(self, collection_path: str) -> pystac.Collection:
-        """
-        Generate a STAC collection from a directory containing the assets to generate metadata
-
-        :param collection_path: path to the collection
-        """
-        # Get the images of the collection, as they are needed to obtain the collection extent
-        collection_images = self._stac_dataframe[
-            self._stac_dataframe["collection"] == collection_path
-        ]["image"]
-        # Get the collection extent
-        extent = get_collection_extent(collection_images)
-        # Create the collection
-        collection_id = basename(collection_path)
-        collection = pystac.Collection(
-            id=collection_id, description="Collection", extent=extent
-        )
-
-        print(f"Generating {collection_id} collection...")
-        for image in tqdm(collection_images):
-            # Create the item
-            item = self.create_stac_item(image)
-            # Add the item to the collection
-            collection.add_item(item)
-
-        # Return the collection
-        return collection
-
-    def create_stac_item(self, raster_path: str) -> pystac.Item:
-        """
-        Create a STAC item from a directory containing the raster files and the metadata.json file
-
-        :param raster_path: path to the raster file
-        """
-        # Check if there is any metadata file in the directory associated to the raster file
-        metadata = get_item_metadata(raster_path)
-
-        # Obtain the bounding box from the raster
-        with rasterio.open(raster_path) as ds:
-            bounds = ds.bounds
-            dst_crs = "EPSG:4326"
-            try:
-                left, bottom, right, top = rasterio.warp.transform_bounds(
-                    ds.crs, dst_crs, *bounds
-                )
-            except rasterio.errors.CRSError:
-                # If the raster has no crs, set the bounding box to 0
-                left, bottom, right, top = 0, 0, 0, 0
-
-        # Create bbox
-        bbox = [left, bottom, right, top]
-
-        # Create geojson feature
-        # If the bounding box has no values, set the geometry to None
-        geom = mapping(
-            Polygon([[left, bottom], [left, top], [right, top], [right, bottom]])
-        )
-
-        # Initialize pySTAC item parameters
-        params = {}
-        params["properties"] = {}
-
-        # Obtain the date acquired
-        start_time, end_time = None, None
-        if (
-            metadata
-            and metadata["acquisition-date"]
-            and metadata["type"] not in ("dem", "DEM")
-        ):
-            time_acquired = format_time_acquired(metadata["acquisition-date"])
-        else:
-            # Check if the type of the data is DEM
-            if metadata and metadata["type"] and metadata["type"] in ("dem", "DEM"):
-                time_acquired = None
-                start_time = datetime.strptime("2011-01-01", "%Y-%m-%d")
-                end_time = datetime.strptime("2015-01-07", "%Y-%m-%d")
-                params["start_datetime"] = start_time
-                params["end_datetime"] = end_time
-            else:
-                # Set unknown date
-                time_acquired = datetime.strptime("2000-01-01", "%Y-%m-%d")
-
-        # Obtain the item ID. The approach depends on the item parser
-        item_id = self._item_parser.get_item_id(raster_path)
-        # Add the item ID to the dataframe, to be able to get it later
-        self._stac_dataframe.loc[
-            self._stac_dataframe["image"] == raster_path, "id"
-        ] = item_id
-
-        # Instantiate pystac item
-        item = pystac.Item(
-            id=item_id, geometry=geom, bbox=bbox, datetime=time_acquired, **params
-        )
-
-        # Get the item info, from the raster path
-        item_info = self._stac_dataframe[self._stac_dataframe["image"] == raster_path]
-        # Get the extensions of the item
-        extensions = item_info["extensions"].values
-        extensions = extensions[0] if extensions else None
-
-        # Add the required extensions to the item
-        if extensions:
-            if isinstance(extensions, str):
-                extensions = [extensions]
-            for extension in extensions:
-                if extension not in SUPPORTED_EXTENSIONS:
-                    raise ValueError(f"Extension {extension} not supported")
-                else:
-                    extension_obj = self._extensions_dict[extension]
-                    extension_obj.add_extension_to_object(item, item_info)
-
-        # Add the assets to the item
-        assets = self._assets_generator.extract_assets(item_info)
-        if not assets:
-            # If there are not assets using the selected generator, try with the default
-            assets = STACAssetGenerator.extract_assets(item_info)
+import pystac
+from datetime import datetime
 
-        # Add the assets to the item
-        if assets:
-            for asset in assets:
-                if isinstance(asset, pystac.Asset):
-                    item.add_asset(asset.title, asset)
-                    # Add the required extensions to the asset if required
-                    if extensions:
-                        if isinstance(extensions, str):
-                            extensions = [extensions]
-                        for extension in extensions:
-                            if extension not in SUPPORTED_EXTENSIONS:
-                                raise ValueError(f"Extension {extension} not supported")
-                            else:
-                                extension_obj = self._extensions_dict[extension]
-                                extension_obj.add_extension_to_object(asset, item_info)
-        item.set_self_href(join(dirname(raster_path), f"{item_id}.json"))
-        item.make_asset_hrefs_relative()
-        return item
+def create_stac_catalog(parquet_catalog_path, stac_catalog = None):
+	# parse items and add to collection
+	table = pq.read_table(parquet_catalog_path)
+	items = []
+	for item in tqdm(stac_geoparquet.arrow.stac_table_to_items(table), total=len(table)):
+		item = pystac.Item.from_dict(item)
+		item.validate()
+		# collection.add_item(item)
+		if stac_catalog is not None:
+			stac_catalog.add_item(item)
+		else:
+			items.append(item)
+		# path = "data/stac/" + item["id"] + ".json"
+		# os.makedirs(os.path.dirname(path), exist_ok=True)
+		# with open(path, "w") as f:
+		# 	json.dump(item, f)
+		# # save item
+		# os.makedirs(path, exist_ok=True)
+		# _path = path + '/' + item.id + ".json"
+		# os.makedirs(os.path.dirname(_path), exist_ok=True)
+		# with open(_path, "w") as f:
+		# 	json.dump(item.to_dict(), f)
+	# save catalog
+	if stac_catalog is not None:
+		return stac_catalog
+	return items

eotdl/datasets/__init__.py

@@ -1,3 +1,3 @@
 from .retrieve import retrieve_datasets, retrieve_dataset, retrieve_dataset_files
 from .ingest import ingest_dataset
-from .download import download_dataset, download_file_url
+from .stage import stage_dataset, stage_dataset_file

eotdl/datasets/ingest.py

@@ -1,167 +1,36 @@
 from pathlib import Path
-import yaml
-from tqdm import tqdm
-import json
-import frontmatter
-
-from ..auth import with_auth
-from .metadata import Metadata
-from ..repos import DatasetsAPIRepo, FilesAPIRepo
-from ..files import ingest_files, create_new_version
-from ..curation.stac import STACDataFrame
-from ..shared import calculate_checksum
-from .update import update_dataset
-from .metadata import generate_metadata
-
-
-def ingest_dataset(
-    path,
-    verbose=False,
-    logger=print,
-    force_metadata_update=False,
-    sync_metadata=False,
-):
-    path = Path(path)
-    if not path.is_dir():
-        raise Exception("Path must be a folder")
-    if "catalog.json" in [f.name for f in path.iterdir()]:
-        return ingest_stac(path / "catalog.json", logger)
-    return ingest_folder(path, verbose, logger, force_metadata_update, sync_metadata)
 
+from ..repos import DatasetsAPIRepo
+from ..files.ingest import prep_ingest_stac, prep_ingest_folder, ingest
 
 def retrieve_dataset(metadata, user):
-    repo = DatasetsAPIRepo()
-    data, error = repo.retrieve_dataset(metadata.name)
-    # print(data, error)
-    if data and data["uid"] != user["uid"]:
-        raise Exception("Dataset already exists.")
-    if error and error == "Dataset doesn't exist":
-        # create dataset
-        data, error = repo.create_dataset(metadata.dict(), user)
-        # print(data, error)
-        if error:
-            raise Exception(error)
-        data["id"] = data["dataset_id"]
-    return data
-
+	repo = DatasetsAPIRepo()
+	data, error = repo.retrieve_dataset(metadata.name)
+	# print(data, error)
+	if data and data["uid"] != user["uid"]:
+		raise Exception("Dataset already exists.")
+	if error and error == "Dataset doesn't exist":
+		# create dataset
+		data, error = repo.create_dataset(metadata.dict(), user)
+		# print(data, error)
+		if error:
+			raise Exception(error)
+	return data
 
-@with_auth
-def ingest_folder(
-    folder,
-    verbose=False,
-    logger=print,
-    force_metadata_update=False,
-    sync_metadata=False,
-    user=None,
-):
-    repo = DatasetsAPIRepo()
-    try:
-        readme = frontmatter.load(folder.joinpath("README.md"))
-        metadata, content = readme.metadata, readme.content
-        metadata = Metadata(**metadata)
-    # except FileNotFoundError:
-    #     # load metadata (legacy)
-    #     metadata = (
-    #         yaml.safe_load(open(folder.joinpath("metadata.yml"), "r").read()) or {}
-    #     )
-    #     metadata = Metadata(**metadata)
-    #     content = None
-    except Exception as e:
-        print(str(e))
-        raise Exception("Error loading metadata")
-    # retrieve dataset (create if doesn't exist)
-    dataset = retrieve_dataset(metadata, user)
-
-    update_metadata = True
-    if "description" in dataset:
-        # do not do this if the dataset is new, only if it already exists
-        update_metadata = check_metadata(
-            dataset, metadata, content, force_metadata_update, sync_metadata, folder
-        )
-    if update_metadata:
-        update_dataset(dataset["id"], metadata, content, user)
-    return ingest_files(
-        repo, dataset["id"], folder, verbose, logger, user, endpoint="datasets"
-    )
-
-
-def check_metadata(
-    dataset, metadata, content, force_metadata_update, sync_metadata, folder
+def ingest_dataset(
+	path,
+	verbose=False,
+	logger=print,
+	force_metadata_update=False,
+	sync_metadata=False,
 ):
-    if (
-        dataset["name"] != metadata.name
-        or dataset["description"] != content
-        or dataset["authors"] != metadata.authors
-        or dataset["source"] != metadata.source
-        or dataset["license"] != metadata.license
-        or dataset["thumbnail"] != metadata.thumbnail
-    ):
-        if not force_metadata_update and not sync_metadata:
-            raise Exception(
-                "The provided metadata is not consistent with the current metadata. Use -f to force metadata update or -s to sync your local metadata."
-            )
-        if force_metadata_update:
-            return True
-        if sync_metadata:
-            generate_metadata(str(folder), dataset)
-            return False
-    return False
-
-
-def retrieve_stac_dataset(dataset_name, user):
-    repo = DatasetsAPIRepo()
-    data, error = repo.retrieve_dataset(dataset_name)
-    # print(data, error)
-    if data and data["uid"] != user["uid"]:
-        raise Exception("Dataset already exists.")
-    if error and error == "Dataset doesn't exist":
-        # create dataset
-        data, error = repo.create_stac_dataset(dataset_name, user)
-        # print(data, error)
-        if error:
-            raise Exception(error)
-        data["id"] = data["dataset_id"]
-    return data["id"]
+	path = Path(path)
+	if not path.is_dir():
+		raise Exception("Path must be a folder")
+	if "catalog.json" in [f.name for f in path.iterdir()]:
+		prep_ingest_stac(path, logger)
+	else:
+		prep_ingest_folder(path, verbose, logger, force_metadata_update, sync_metadata)
+	return ingest(path, DatasetsAPIRepo(), retrieve_dataset, 'datasets')
 
 
-@with_auth
-def ingest_stac(stac_catalog, logger=None, user=None):
-    repo, files_repo = DatasetsAPIRepo(), FilesAPIRepo()
-    # load catalog
-    logger("Loading STAC catalog...")
-    df = STACDataFrame.from_stac_file(stac_catalog)
-    catalog = df[df["type"] == "Catalog"]
-    assert len(catalog) == 1, "STAC catalog must have exactly one root catalog"
-    dataset_name = catalog.id.iloc[0]
-    # retrieve dataset (create if doesn't exist)
-    dataset_id = retrieve_stac_dataset(dataset_name, user)
-    # create new version
-    version = create_new_version(repo, dataset_id, user)
-    logger("New version created, version: " + str(version))
-    df2 = df.dropna(subset=["assets"])
-    for row in tqdm(df2.iterrows(), total=len(df2)):
-        try:
-            for k, v in row[1]["assets"].items():
-                data, error = files_repo.ingest_file(
-                    v["href"],
-                    dataset_id,
-                    user,
-                    calculate_checksum(v["href"]),  # is always absolute?
-                    "datasets",
-                    version,
-                )
-                if error:
-                    raise Exception(error)
-                file_url = f"{repo.url}datasets/{data['dataset_id']}/download/{data['filename']}"
-                df.loc[row[0], "assets"][k]["href"] = file_url
-        except Exception as e:
-            logger(f"Error uploading asset {row[0]}: {e}")
-            break
-    # ingest the STAC catalog into geodb
-    logger("Ingesting STAC catalog...")
-    data, error = repo.ingest_stac(json.loads(df.to_json()), dataset_id, user)
-    if error:
-        # TODO: delete all assets that were uploaded
-        raise Exception(error)
-    logger("Done")
-    return

eotdl/datasets/retrieve.py

@@ -25,12 +25,3 @@ def retrieve_dataset_files(dataset_id, version):
         raise Exception(error)
     return data
 
-
-# def list_datasets(pattern=None):
-#     datasets = retrieve_datasets()
-#     if pattern:
-#         regex = re.compile(rf".*{re.escape(pattern)}.*", re.IGNORECASE)
-#         names = list(datasets.keys())
-#         valid = [name for name in names if regex.search(name)]
-#         return {name: datasets[name] for name in valid}
-#     return datasets

eotdl/datasets/stage.py

@@ -0,0 +1,64 @@
+import os
+from pathlib import Path
+from tqdm import tqdm
+import geopandas as gpd
+
+from ..auth import with_auth
+from .retrieve import retrieve_dataset
+from ..repos import FilesAPIRepo
+
+@with_auth
+def stage_dataset(
+    dataset_name,
+    version=None,
+    path=None,
+    logger=print,
+    assets=False,
+    force=False,
+    verbose=False,
+    user=None,
+    file=None,
+):
+    dataset = retrieve_dataset(dataset_name)
+    if version is None:
+        version = sorted([v['version_id'] for v in dataset["versions"]])[-1]
+    else:
+        assert version in [
+            v["version_id"] for v in dataset["versions"]
+        ], f"Version {version} not found"
+    download_base_path = os.getenv(
+        "EOTDL_DOWNLOAD_PATH", str(Path.home()) + "/.cache/eotdl/datasets"
+    )
+    if path is None:
+        download_path = download_base_path + "/" + dataset_name 
+    else:
+        download_path = path + "/" + dataset_name 
+    # check if dataset already exists
+    if os.path.exists(download_path) and not force:
+        os.makedirs(download_path, exist_ok=True)
+        # raise Exception(
+        #     f"Dataset `{dataset['name']} v{str(version)}` already exists at {download_path}. To force download, use force=True or -f in the CLI."
+        # )
+        raise Exception(
+            f"Dataset `{dataset['name']}` already exists at {download_path}. To force download, use force=True or -f in the CLI."
+        )
+
+    # stage metadata
+    repo = FilesAPIRepo()
+    catalog_path = repo.stage_file(dataset["id"], f"catalog.v{version}.parquet", user, download_path)
+
+    # TODO: stage README.md
+
+    if assets:
+        gdf = gpd.read_parquet(catalog_path)
+        for _, row in tqdm(gdf.iterrows(), total=len(gdf), desc="Staging assets"):
+            for k, v in row["assets"].items():
+                stage_dataset_file(v["href"], download_path)
+
+    return download_path
+
+
+@with_auth
+def stage_dataset_file(file_url, path, user):
+    repo = FilesAPIRepo()
+    return repo.stage_file_url(file_url, path, user)

eotdl/files/__init__.py

@@ -1,2 +0,0 @@
-from .ingest import ingest_files, create_new_version
-from .list_files import list_files