mapchete-eo 2026.2.0__py3-none-any.whl

mapchete_eo/cli/s2_verify.py

@@ -0,0 +1,129 @@
+import logging
+from dataclasses import dataclass
+from typing import List, Optional
+
+import click
+import numpy as np
+import pystac
+from mapchete.cli.options import opt_debug
+from mapchete.io import copy
+from mapchete.io.raster import read_raster_no_crs
+from mapchete.path import MPath
+from tqdm import tqdm
+
+from mapchete_eo.array.color import outlier_pixels
+from mapchete_eo.cli import options_arguments
+from mapchete_eo.exceptions import AssetKeyError
+from mapchete_eo.platforms.sentinel2.product import asset_mpath
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class Report:
+    item: pystac.Item
+    missing_asset_entries: List[str]
+    missing_assets: List[MPath]
+    color_artefacts: bool = False
+
+    def product_broken(self) -> bool:
+        return any(
+            [
+                bool(self.missing_asset_entries),
+                bool(self.missing_assets),
+                bool(self.color_artefacts),
+            ]
+        )
+
+
+@click.command()
+@options_arguments.arg_stac_items
+@options_arguments.opt_assets
+@opt_debug
+def s2_verify(
+    stac_items: List[MPath],
+    assets: List[str] = [],
+    asset_exists_check: bool = True,
+    **_,
+):
+    """Verify Sentinel-2 products."""
+    assets = assets or []
+    for item_path in tqdm(stac_items):
+        report = verify_item(
+            pystac.Item.from_file(item_path),
+            assets=assets,
+            asset_exists_check=asset_exists_check,
+        )
+        for asset in report.missing_asset_entries:
+            tqdm.write(f"[ERROR] {report.item.id} has no asset named '{asset}")
+        for path in report.missing_assets:
+            tqdm.write(
+                f"[ERROR] {report.item.id} asset '{asset}' with path {str(path)} does not exist"
+            )
+        if report.color_artefacts:
+            tqdm.write(
+                f"[ERROR] {report.item.id} thumbnail ({report.item.assets['thumbnail'].href}) indicates that there are some color artefacts"
+            )
+
+
+def verify_item(
+    item: pystac.Item,
+    assets: List[str],
+    asset_exists_check: bool = False,
+    check_thumbnail: bool = True,
+    thumbnail_dir: Optional[MPath] = None,
+):
+    missing_asset_entries = []
+    missing_assets = []
+    color_artefacts = False
+    for asset in assets:
+        logger.debug("verify asset %s is available", asset)
+        if asset not in item.assets:
+            missing_asset_entries.append(asset)
+        if asset_exists_check:
+            try:
+                path = asset_mpath(item=item, asset=asset)
+                logger.debug("check if asset %s (%s) exists", asset, str(path))
+                if not path.exists():
+                    missing_assets.append(path)
+            except AssetKeyError:
+                missing_asset_entries.append(asset)
+    if check_thumbnail:
+        thumbnail_href = MPath.from_inp(item.assets["thumbnail"].href)
+        logger.debug("check thumbnail %s for artefacts ...", thumbnail_href)
+        if thumbnail_dir:
+            thumbnail_path = thumbnail_dir / item.id + ".jpg"
+            copy(thumbnail_href, thumbnail_path)
+        else:
+            thumbnail_path = thumbnail_href
+        color_artefacts = outlier_pixels_detected(read_raster_no_crs(thumbnail_href))
+    return Report(
+        item,
+        missing_asset_entries=missing_asset_entries,
+        missing_assets=missing_assets,
+        color_artefacts=color_artefacts,
+    )
+
+
+def outlier_pixels_detected(
+    arr: np.ndarray,
+    axis: int = 0,
+    range_threshold: int = 100,
+    allowed_error_percentage: float = 1,
+) -> bool:
+    """
+    Checks whether number of outlier pixels is larger than allowed.
+
+    An outlier pixel is a pixel, where the value range between bands exceeds
+    the range_threshold.
+    """
+    _, width, height = arr.shape
+    pixels = width * height
+    outliers = outlier_pixels(arr, axis=axis, range_threshold=range_threshold).sum()
+    outlier_percent = outliers / pixels * 100
+    logger.debug(
+        "%s (%s %%) suspicious pixels detected",
+        outliers,
+        round(outlier_percent, 2),
+    )
+    return outlier_percent > allowed_error_percentage

mapchete_eo/cli/static_catalog.py

@@ -0,0 +1,82 @@
+from datetime import datetime
+from typing import List, Optional
+
+import click
+from mapchete.cli.options import opt_bounds, opt_debug
+from mapchete.path import MPath
+from mapchete.types import Bounds
+from rasterio.profiles import Profile
+
+from mapchete_eo.cli import options_arguments
+from mapchete_eo.platforms.sentinel2 import S2Metadata
+from mapchete_eo.platforms.sentinel2.source import Sentinel2Source
+from mapchete_eo.platforms.sentinel2.types import Resolution
+from mapchete_eo.types import TimeRange
+
+
+@click.command()
+@options_arguments.arg_dst_path
+@opt_bounds
+@options_arguments.opt_mgrs_tile
+@options_arguments.opt_start_time
+@options_arguments.opt_end_time
+@options_arguments.opt_source
+@options_arguments.opt_name
+@options_arguments.opt_description
+@options_arguments.opt_assets
+@options_arguments.opt_assets_dst_resolution
+@options_arguments.opt_assets_dst_rio_profile
+@options_arguments.opt_copy_metadata
+@options_arguments.opt_overwrite
+@opt_debug
+def static_catalog(
+    dst_path: MPath,
+    start_time: datetime,
+    end_time: datetime,
+    bounds: Optional[Bounds] = None,
+    mgrs_tile: Optional[str] = None,
+    source: Sentinel2Source = Sentinel2Source(collection="EarthSearch"),
+    name: Optional[str] = None,
+    description: Optional[str] = None,
+    assets: Optional[List[str]] = None,
+    assets_dst_resolution: Resolution = Resolution.original,
+    assets_dst_rio_profile: Optional[Profile] = None,
+    copy_metadata: bool = False,
+    overwrite: bool = False,
+    **__,
+):
+    """Write a static STAC catalog for selected area."""
+    if any([start_time is None, end_time is None]):  # pragma: no cover
+        raise click.ClickException("--start-time and --end-time are mandatory")
+    if all([bounds is None, mgrs_tile is None]):  # pragma: no cover
+        raise click.ClickException("--bounds or --mgrs-tile are required")
+    catalog = source.get_catalog()
+    if hasattr(catalog, "write_static_catalog"):
+        with options_arguments.TqdmUpTo(
+            unit="products", unit_scale=True, miniters=1, disable=opt_debug
+        ) as progress:
+            catalog_json = catalog.write_static_catalog(
+                dst_path,
+                name=name,
+                bounds=bounds,
+                time=TimeRange(
+                    start=start_time,
+                    end=end_time,
+                ),
+                search_kwargs=dict(mgrs_tile=mgrs_tile),
+                description=description,
+                assets=assets,
+                assets_dst_resolution=assets_dst_resolution.value,
+                assets_convert_profile=assets_dst_rio_profile,
+                copy_metadata=copy_metadata,
+                metadata_parser_classes=(S2Metadata,),
+                overwrite=overwrite,
+                progress_callback=progress.update_to,
+            )
+
+        click.echo(f"Catalog successfully written to {catalog_json}")
+
+    else:
+        raise AttributeError(
+            f"catalog {catalog} does not support writing a static version"
+        )

mapchete_eo/eostac.py

@@ -0,0 +1,30 @@
+"""
+Driver class for EOSTAC static STAC catalogs.
+"""
+
+from mapchete_eo import base
+
+METADATA: dict = {
+    "driver_name": "EOSTAC",
+    "data_type": None,
+    "mode": "r",
+    "file_extensions": [],
+}
+
+
+class InputTile(base.EODataCube):
+    """
+    Target Tile representation of input data.
+
+    Parameters
+    ----------
+    tile : ``Tile``
+    kwargs : keyword arguments
+        driver specific parameters
+    """
+
+
+class InputData(base.InputData):
+    """In case this driver is used when being a readonly input to another process."""
+
+    input_tile_cls = InputTile

mapchete_eo/exceptions.py

@@ -0,0 +1,87 @@
+"""Custom exceptions."""
+
+from mapchete.errors import MapcheteNodataTile
+
+
+class EmptyFootprintException(Exception):
+    """Raised when footprint is empty."""
+
+
+class EmptySliceException(Exception):
+    """Raised when slice is empty."""
+
+
+class EmptyProductException(EmptySliceException):
+    """Raised when product is empty."""
+
+
+class EmptyStackException(MapcheteNodataTile):
+    """Raised when whole stack is empty."""
+
+
+class EmptyFileException(Exception):
+    """Raised when no bytes are downloaded."""
+
+
+class IncompleteDownloadException(Exception):
+    """ "Raised when the file is not downloaded completely."""
+
+
+class InvalidMapcheteEOCollectionError(Exception):
+    """ "Raised for unsupported collections of Mapchete EO package."""
+
+
+class EmptyCatalogueResponse(Exception):
+    """Raised when catalogue response is empty."""
+
+
+class CorruptedGTiffError(Exception):
+    """Raised when GTiff validation fails."""
+
+
+class BRDFError(Exception):
+    """Raised when BRDF grid cannot be calculated."""
+
+
+class AssetError(Exception):
+    """Generic Exception class for Assets."""
+
+
+class AssetMissing(AssetError, FileNotFoundError):
+    """Raised when a product asset should be there but isn't."""
+
+
+class AssetEmpty(AssetError):
+    """Raised when a product asset should contain data but is empty."""
+
+
+class AssetKeyError(AssetError, KeyError):
+    """Raised when an asset name cannot be found in item."""
+
+
+class PreprocessingNotFinished(Exception):
+    """Raised when preprocessing tasks have not been fully executed."""
+
+
+class AllMasked(Exception):
+    """Raised when an array is fully masked."""
+
+
+class NoSourceProducts(MapcheteNodataTile, ValueError):
+    """Raised when no products are available."""
+
+
+class CorruptedProduct(Exception):
+    """Raised when product is damaged and cannot be read."""
+
+
+class CorruptedProductMetadata(CorruptedProduct):
+    """Raised when EOProduct cannot be parsed due to a metadata issue."""
+
+
+class CorruptedSlice(Exception):
+    """Raised when all products in a slice are damaged and cannot be read."""
+
+
+class ItemGeometryError(Exception):
+    """Raised when STAC item geometry cannot be resolved."""

mapchete_eo/image_operations/__init__.py

@@ -0,0 +1,12 @@
+from mapchete_eo.image_operations.color_correction import color_correct
+from mapchete_eo.image_operations.dtype_scale import dtype_scale
+from mapchete_eo.image_operations.fillnodata import FillSelectionMethod, fillnodata
+from mapchete_eo.image_operations.linear_normalization import linear_normalization
+
+__all__ = [
+    "color_correct",
+    "dtype_scale",
+    "fillnodata",
+    "FillSelectionMethod",
+    "linear_normalization",
+]