mapchete-eo 2025.7.0__py2.py3-none-any.whl

mapchete_eo/cli/s2_verify.py

@@ -0,0 +1,129 @@
+import logging
+from dataclasses import dataclass
+from typing import List, Optional
+
+import click
+import numpy as np
+import pystac
+from mapchete.cli.options import opt_debug
+from mapchete.io import copy
+from mapchete.io.raster import read_raster_no_crs
+from mapchete.path import MPath
+from tqdm import tqdm
+
+from mapchete_eo.array.color import outlier_pixels
+from mapchete_eo.cli import options_arguments
+from mapchete_eo.exceptions import AssetKeyError
+from mapchete_eo.platforms.sentinel2.product import asset_mpath
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class Report:
+    item: pystac.Item
+    missing_asset_entries: List[str]
+    missing_assets: List[MPath]
+    color_artefacts: bool = False
+
+    def product_broken(self) -> bool:
+        return any(
+            [
+                bool(self.missing_asset_entries),
+                bool(self.missing_assets),
+                bool(self.color_artefacts),
+            ]
+        )
+
+
+@click.command()
+@options_arguments.arg_stac_items
+@options_arguments.opt_assets
+@opt_debug
+def s2_verify(
+    stac_items: List[MPath],
+    assets: List[str] = [],
+    asset_exists_check: bool = True,
+    **_,
+):
+    """Verify Sentinel-2 products."""
+    assets = assets or []
+    for item_path in tqdm(stac_items):
+        report = verify_item(
+            pystac.Item.from_file(item_path),
+            assets=assets,
+            asset_exists_check=asset_exists_check,
+        )
+        for asset in report.missing_asset_entries:
+            tqdm.write(f"[ERROR] {report.item.id} has no asset named '{asset}")
+        for path in report.missing_assets:
+            tqdm.write(
+                f"[ERROR] {report.item.id} asset '{asset}' with path {str(path)} does not exist"
+            )
+        if report.color_artefacts:
+            tqdm.write(
+                f"[ERROR] {report.item.id} thumbnail ({report.item.assets['thumbnail'].href}) indicates that there are some color artefacts"
+            )
+
+
+def verify_item(
+    item: pystac.Item,
+    assets: List[str],
+    asset_exists_check: bool = False,
+    check_thumbnail: bool = True,
+    thumbnail_dir: Optional[MPath] = None,
+):
+    missing_asset_entries = []
+    missing_assets = []
+    color_artefacts = False
+    for asset in assets:
+        logger.debug("verify asset %s is available", asset)
+        if asset not in item.assets:
+            missing_asset_entries.append(asset)
+        if asset_exists_check:
+            try:
+                path = asset_mpath(item=item, asset=asset)
+                logger.debug("check if asset %s (%s) exists", asset, str(path))
+                if not path.exists():
+                    missing_assets.append(path)
+            except AssetKeyError:
+                missing_asset_entries.append(asset)
+    if check_thumbnail:
+        thumbnail_href = MPath.from_inp(item.assets["thumbnail"].href)
+        logger.debug("check thumbnail %s for artefacts ...", thumbnail_href)
+        if thumbnail_dir:
+            thumbnail_path = thumbnail_dir / item.id + ".jpg"
+            copy(thumbnail_href, thumbnail_path)
+        else:
+            thumbnail_path = thumbnail_href
+        color_artefacts = outlier_pixels_detected(read_raster_no_crs(thumbnail_href))
+    return Report(
+        item,
+        missing_asset_entries=missing_asset_entries,
+        missing_assets=missing_assets,
+        color_artefacts=color_artefacts,
+    )
+
+
+def outlier_pixels_detected(
+    arr: np.ndarray,
+    axis: int = 0,
+    range_threshold: int = 100,
+    allowed_error_percentage: float = 1,
+) -> bool:
+    """
+    Checks whether number of outlier pixels is larger than allowed.
+
+    An outlier pixel is a pixel, where the value range between bands exceeds
+    the range_threshold.
+    """
+    _, width, height = arr.shape
+    pixels = width * height
+    outliers = outlier_pixels(arr, axis=axis, range_threshold=range_threshold).sum()
+    outlier_percent = outliers / pixels * 100
+    logger.debug(
+        "%s (%s %%) suspicious pixels detected",
+        outliers,
+        round(outlier_percent, 2),
+    )
+    return outlier_percent > allowed_error_percentage

mapchete_eo/cli/static_catalog.py

@@ -0,0 +1,123 @@
+from datetime import datetime
+from typing import List, Optional
+
+import click
+from mapchete.cli.options import opt_bounds, opt_debug
+from mapchete.path import MPath
+from mapchete.types import Bounds
+from rasterio.profiles import Profile
+
+from mapchete_eo.cli import options_arguments
+from mapchete_eo.platforms.sentinel2 import S2Metadata
+from mapchete_eo.platforms.sentinel2.archives import KnownArchives
+from mapchete_eo.platforms.sentinel2.types import Resolution
+from mapchete_eo.search import STACSearchCatalog, STACStaticCatalog
+from mapchete_eo.search.base import CatalogSearcher
+from mapchete_eo.types import TimeRange
+
+
+@click.command()
+@options_arguments.arg_dst_path
+@opt_bounds
+@options_arguments.opt_mgrs_tile
+@options_arguments.opt_start_time
+@options_arguments.opt_end_time
+@options_arguments.opt_archive
+@options_arguments.opt_collection
+@options_arguments.opt_endpoint
+@options_arguments.opt_catalog_json
+@options_arguments.opt_name
+@options_arguments.opt_description
+@options_arguments.opt_assets
+@options_arguments.opt_assets_dst_resolution
+@options_arguments.opt_assets_dst_rio_profile
+@options_arguments.opt_copy_metadata
+@options_arguments.opt_overwrite
+@opt_debug
+def static_catalog(
+    dst_path: MPath,
+    start_time: datetime,
+    end_time: datetime,
+    bounds: Optional[Bounds] = None,
+    mgrs_tile: Optional[str] = None,
+    archive: Optional[KnownArchives] = None,
+    collection: Optional[str] = None,
+    endpoint: Optional[str] = None,
+    catalog_json: Optional[MPath] = None,
+    name: Optional[str] = None,
+    description: Optional[str] = None,
+    assets: Optional[List[str]] = None,
+    assets_dst_resolution: Resolution = Resolution.original,
+    assets_dst_rio_profile: Optional[Profile] = None,
+    copy_metadata: bool = False,
+    overwrite: bool = False,
+    **__,
+):
+    """Write a static STAC catalog for selected area."""
+    if catalog_json and endpoint:  # pragma: no cover
+        raise click.ClickException(
+            "exactly one of --archive, --catalog-json or --endpoint has to be set."
+        )
+    if any([start_time is None, end_time is None]):  # pragma: no cover
+        raise click.ClickException("--start-time and --end-time are mandatory")
+    if all([bounds is None, mgrs_tile is None]):  # pragma: no cover
+        raise click.ClickException("--bounds or --mgrs-tile are required")
+    catalog = get_catalog(
+        catalog_json=catalog_json,
+        endpoint=endpoint,
+        known_archive=archive,
+        collection=collection,
+    )
+    if hasattr(catalog, "write_static_catalog"):
+        with options_arguments.TqdmUpTo(
+            unit="products", unit_scale=True, miniters=1, disable=opt_debug
+        ) as progress:
+            catalog_json = catalog.write_static_catalog(
+                dst_path,
+                name=name,
+                bounds=bounds,
+                time=TimeRange(
+                    start=start_time,
+                    end=end_time,
+                ),
+                search_kwargs=dict(mgrs_tile=mgrs_tile),
+                description=description,
+                assets=assets,
+                assets_dst_resolution=assets_dst_resolution.value,
+                assets_convert_profile=assets_dst_rio_profile,
+                copy_metadata=copy_metadata,
+                metadata_parser_classes=(S2Metadata,),
+                overwrite=overwrite,
+                progress_callback=progress.update_to,
+            )
+
+        click.echo(f"Catalog successfully written to {catalog_json}")
+
+    else:
+        raise AttributeError(
+            f"catalog {catalog} does not support writing a static version"
+        )
+
+
+def get_catalog(
+    catalog_json: Optional[MPath],
+    endpoint: Optional[MPath],
+    known_archive: Optional[KnownArchives] = None,
+    collection: Optional[str] = None,
+) -> CatalogSearcher:
+    if catalog_json:
+        return STACStaticCatalog(
+            baseurl=catalog_json,
+        )
+    elif endpoint:
+        if collection:
+            return STACSearchCatalog(
+                endpoint=endpoint,
+                collections=[collection],
+            )
+        else:
+            raise ValueError("collection must be provided")
+    elif known_archive:
+        return known_archive.value.catalog
+    else:
+        raise TypeError("cannot determine catalog")

mapchete_eo/eostac.py

@@ -0,0 +1,30 @@
+"""
+Driver class for EOSTAC static STAC catalogs.
+"""
+
+from mapchete_eo import base
+
+METADATA: dict = {
+    "driver_name": "EOSTAC_DEV",
+    "data_type": None,
+    "mode": "r",
+    "file_extensions": [],
+}
+
+
+class InputTile(base.EODataCube):
+    """
+    Target Tile representation of input data.
+
+    Parameters
+    ----------
+    tile : ``Tile``
+    kwargs : keyword arguments
+        driver specific parameters
+    """
+
+
+class InputData(base.InputData):
+    """In case this driver is used when being a readonly input to another process."""
+
+    input_tile_cls = InputTile

mapchete_eo/exceptions.py

@@ -0,0 +1,87 @@
+"""Custom exceptions."""
+
+from mapchete.errors import MapcheteNodataTile
+
+
+class EmptyFootprintException(Exception):
+    """Raised when footprint is empty."""
+
+
+class EmptySliceException(Exception):
+    """Raised when slice is empty."""
+
+
+class EmptyProductException(EmptySliceException):
+    """Raised when product is empty."""
+
+
+class EmptyStackException(MapcheteNodataTile):
+    """Raised when whole stack is empty."""
+
+
+class EmptyFileException(Exception):
+    """Raised when no bytes are downloaded."""
+
+
+class IncompleteDownloadException(Exception):
+    """ "Raised when the file is not downloaded completely."""
+
+
+class InvalidMapcheteEOCollectionError(Exception):
+    """ "Raised for unsupported collections of Mapchete EO package."""
+
+
+class EmptyCatalogueResponse(Exception):
+    """Raised when catalogue response is empty."""
+
+
+class CorruptedGTiffError(Exception):
+    """Raised when GTiff validation fails."""
+
+
+class BRDFError(Exception):
+    """Raised when BRDF grid cannot be calculated."""
+
+
+class AssetError(Exception):
+    """Generic Exception class for Assets."""
+
+
+class AssetMissing(AssetError, FileNotFoundError):
+    """Raised when a product asset should be there but isn't."""
+
+
+class AssetEmpty(AssetError):
+    """Raised when a product asset should contain data but is empty."""
+
+
+class AssetKeyError(AssetError, KeyError):
+    """Raised when an asset name cannot be found in item."""
+
+
+class PreprocessingNotFinished(Exception):
+    """Raised when preprocessing tasks have not been fully executed."""
+
+
+class AllMasked(Exception):
+    """Raised when an array is fully masked."""
+
+
+class NoSourceProducts(MapcheteNodataTile, ValueError):
+    """Raised when no products are available."""
+
+
+class CorruptedProduct(Exception):
+    """Raised when product is damaged and cannot be read."""
+
+
+class CorruptedProductMetadata(CorruptedProduct):
+    """Raised when EOProduct cannot be parsed due to a metadata issue."""
+
+
+class CorruptedSlice(Exception):
+    """Raised when all products in a slice are damaged and cannot be read."""
+
+
+class ItemGeometryError(Exception):
+    """Raised when STAC item geometry cannot be resolved."""

mapchete_eo/geometry.py

@@ -0,0 +1,271 @@
+import logging
+import math
+from functools import partial
+from typing import Callable, Iterable, Tuple
+
+from fiona.crs import CRS
+from fiona.transform import transform as fiona_transform
+from mapchete.geometry import reproject_geometry
+from mapchete.types import Bounds, CRSLike
+from shapely.geometry import (
+    GeometryCollection,
+    LinearRing,
+    LineString,
+    MultiLineString,
+    MultiPoint,
+    MultiPolygon,
+    Point,
+    Polygon,
+    box,
+    shape,
+)
+from shapely.geometry.base import BaseGeometry
+from shapely.ops import unary_union
+
+CoordArrays = Tuple[Iterable[float], Iterable[float]]
+
+
+logger = logging.getLogger(__name__)
+
+
+def transform_to_latlon(
+    geometry: BaseGeometry, src_crs: CRSLike, width_threshold: float = 180.0
+) -> BaseGeometry:
+    """Transforms a geometry to lat/lon coordinates.
+
+    If resulting geometry crosses the Antimeridian it will be fixed by moving coordinates
+    from the Western Hemisphere to outside of the lat/lon bounds on the East, making sure
+    the correct geometry shape is preserved.
+
+    As a next step, repair_antimeridian_geometry() can be applied, which then splits up
+    this geometry into a multipart geometry where all of its subgeometries are within the
+    lat/lon bounds again.
+    """
+    latlon_crs = CRS.from_epsg(4326)
+
+    def transform_shift_coords(coords: CoordArrays) -> CoordArrays:
+        out_x_coords, out_y_coords = fiona_transform(src_crs, latlon_crs, *coords)
+        if max(out_x_coords) - min(out_x_coords) > width_threshold:
+            # we probably have an antimeridian crossing here!
+            out_x_coords, out_y_coords = coords_longitudinal_shift(
+                coords_transform(coords, src_crs, latlon_crs), only_negative_coords=True
+            )
+        return (out_x_coords, out_y_coords)
+
+    return custom_transform(geometry, transform_shift_coords)
+
+
+def repair_antimeridian_geometry(
+    geometry: BaseGeometry, width_threshold: float = 180.0
+) -> BaseGeometry:
+    """
+    Repair geometry and apply fix if it crosses the Antimeridian.
+
+    A geometry crosses the Antimeridian if it is at least partly outside of the
+    lat/lon bounding box or if its width exceeds a certain threshold. This can happen
+    after reprojection if the geometry coordinates are transformed separately and land
+    left and right of the Antimeridian, thus resulting in a polygon spanning almost the
+    whole lat/lon bounding box width.
+    """
+    # repair geometry if it is broken
+    geometry = geometry.buffer(0)
+    latlon_bbox = box(-180, -90, 180, 90)
+
+    # only attempt to fix if geometry is too wide or reaches over the lat/lon bounds
+    if (
+        Bounds.from_inp(geometry).width >= width_threshold
+        or not geometry.difference(latlon_bbox).is_empty
+    ):
+        # (1) shift only coordinates on the western hemisphere by 360°, thus "fixing"
+        # the footprint, but letting it cross the antimeridian
+        shifted_geometry = longitudinal_shift(geometry, only_negative_coords=True)
+
+        # (2) split up geometry in one outside of latlon bounds and one inside
+        inside = shifted_geometry.intersection(latlon_bbox)
+        outside = shifted_geometry.difference(latlon_bbox)
+
+        # (3) shift back only the polygon outside of latlon bounds by -360, thus moving
+        # it back to the western hemisphere
+        outside_shifted = longitudinal_shift(
+            outside, offset=-360, only_negative_coords=False
+        )
+
+        # (4) create a MultiPolygon out from these two polygons
+        geometry = unary_union([inside, outside_shifted])
+
+    return geometry
+
+
+def buffer_antimeridian_safe(
+    footprint: BaseGeometry, buffer_m: float = 0
+) -> BaseGeometry:
+    """Buffer geometry by meters and make it Antimeridian-safe.
+
+    Safe means that if it crosses the Antimeridian and is a MultiPolygon,
+    the buffer will only be applied to the edges facing away from the Antimeridian
+    thus leaving the polygon intact if shifted back.
+    """
+    if footprint.is_empty:
+        return footprint
+
+    # repair geometry if it is broken
+    footprint = footprint.buffer(0)
+
+    if not buffer_m:
+        return footprint
+
+    if isinstance(footprint, MultiPolygon):
+        # we have a shifted footprint here!
+        # (1) unshift one part
+        subpolygons = []
+        for polygon in footprint.geoms:
+            lon = polygon.centroid.x
+            if lon < 0:
+                polygon = longitudinal_shift(polygon)
+            subpolygons.append(polygon)
+        # (2) merge to single polygon
+        merged = unary_union(subpolygons)
+
+        # (3) apply buffer
+        if isinstance(merged, MultiPolygon):
+            buffered = unary_union(
+                [
+                    buffer_antimeridian_safe(polygon, buffer_m=buffer_m)
+                    for polygon in merged.geoms
+                ]
+            )
+        else:
+            buffered = buffer_antimeridian_safe(merged, buffer_m=buffer_m)
+
+        # (4) fix again
+        return repair_antimeridian_geometry(buffered)
+
+    # UTM zone CRS
+    utm_crs = latlon_to_utm_crs(footprint.centroid.y, footprint.centroid.x)
+    latlon_crs = CRS.from_string("EPSG:4326")
+
+    return transform_to_latlon(
+        reproject_geometry(
+            footprint, src_crs=latlon_crs, dst_crs=utm_crs, clip_to_crs_bounds=False
+        ).buffer(buffer_m),
+        src_crs=utm_crs,
+    )
+
+
+def longitudinal_shift(
+    geometry: BaseGeometry, offset: float = 360.0, only_negative_coords: bool = False
+) -> BaseGeometry:
+    """Return geometry with either all or Western hemisphere coordinates shifted by some offset."""
+    return custom_transform(
+        geometry,
+        partial(
+            coords_longitudinal_shift,
+            by=offset,
+            only_negative_coords=only_negative_coords,
+        ),
+    )
+
+
+def latlon_to_utm_crs(lat: float, lon: float) -> CRS:
+    min_zone = 1
+    max_zone = 60
+    utm_zone = (
+        f"{max([min([(math.floor((lon + 180) / 6) + 1), max_zone]), min_zone]):02}"
+    )
+    hemisphere_code = "7" if lat <= 0 else "6"
+    return CRS.from_string(f"EPSG:32{hemisphere_code}{utm_zone}")
+
+
+def bounds_to_geom(bounds: Bounds) -> BaseGeometry:
+    # TODO: move into core package
+    if bounds.left < -180:
+        part1 = Bounds(-180, bounds.bottom, bounds.right, bounds.top)
+        part2 = Bounds(bounds.left + 360, bounds.bottom, 180, bounds.top)
+        return unary_union([shape(part1), shape(part2)])
+    elif bounds.right > 180:
+        part1 = Bounds(-180, bounds.bottom, bounds.right - 360, bounds.top)
+        part2 = Bounds(bounds.left, bounds.bottom, 180, bounds.top)
+        return unary_union([shape(part1), shape(part2)])
+    else:
+        return shape(bounds)
+
+
+def custom_transform(geometry: BaseGeometry, func: Callable) -> BaseGeometry:
+    # todo: shapely.transform.transform maybe can make this code more simple
+    # https://shapely.readthedocs.io/en/stable/reference/shapely.transform.html#shapely.transform
+    def _point(point: Point) -> Point:
+        return Point(zip(*func(point.xy)))
+
+    def _multipoint(multipoint: MultiPoint) -> MultiPoint:
+        return MultiPoint([_point(point) for point in multipoint])
+
+    def _linestring(linestring: LineString) -> LineString:
+        return LineString(zip(*func(linestring.xy)))
+
+    def _multilinestring(multilinestring: MultiLineString) -> MultiLineString:
+        return MultiLineString(
+            [_linestring(linestring) for linestring in multilinestring.geoms]
+        )
+
+    def _linearring(linearring: LinearRing) -> LinearRing:
+        return LinearRing(((x, y) for x, y in zip(*func(linearring.xy))))
+
+    def _polygon(polygon: Polygon) -> Polygon:
+        return Polygon(
+            _linearring(polygon.exterior),
+            holes=list(map(_linearring, polygon.interiors)),
+        )
+
+    def _multipolygon(multipolygon: MultiPolygon) -> MultiPolygon:
+        return MultiPolygon([_polygon(polygon) for polygon in multipolygon.geoms])
+
+    def _geometrycollection(
+        geometrycollection: GeometryCollection,
+    ) -> GeometryCollection:
+        return GeometryCollection(
+            [_any_geometry(subgeometry) for subgeometry in geometrycollection.geoms]
+        )
+
+    def _any_geometry(geometry: BaseGeometry) -> BaseGeometry:
+        transform_funcs = {
+            Point: _point,
+            MultiPoint: _multipoint,
+            LineString: _linestring,
+            MultiLineString: _multilinestring,
+            Polygon: _polygon,
+            MultiPolygon: _multipolygon,
+            GeometryCollection: _geometrycollection,
+        }
+        try:
+            return transform_funcs[type(geometry)](geometry)
+        except KeyError:
+            raise TypeError(f"unknown geometry {geometry} of type {type(geometry)}")
+
+    if geometry.is_empty:
+        return geometry
+
+    # make valid by buffering
+    return _any_geometry(geometry).buffer(0)
+
+
+def coords_transform(
+    coords: CoordArrays, src_crs: CRSLike, dst_crs: CRSLike
+) -> CoordArrays:
+    return fiona_transform(src_crs, dst_crs, *coords)
+
+
+def coords_longitudinal_shift(
+    coords: CoordArrays,
+    by: float = 360,
+    only_negative_coords: bool = False,
+) -> CoordArrays:
+    x_coords, y_coords = coords
+    x_coords = (
+        (
+            x_coord + by
+            if (only_negative_coords and x_coord < 0) or not only_negative_coords
+            else x_coord
+        )
+        for x_coord in x_coords
+    )
+    return x_coords, y_coords

mapchete_eo/image_operations/__init__.py

@@ -0,0 +1,12 @@
+from mapchete_eo.image_operations.color_correction import color_correct
+from mapchete_eo.image_operations.dtype_scale import dtype_scale
+from mapchete_eo.image_operations.fillnodata import FillSelectionMethod, fillnodata
+from mapchete_eo.image_operations.linear_normalization import linear_normalization
+
+__all__ = [
+    "color_correct",
+    "dtype_scale",
+    "fillnodata",
+    "FillSelectionMethod",
+    "linear_normalization",
+]