mapchete-eo 2026.2.0__py3-none-any.whl

mapchete_eo/cli/s2_cat_results.py

@@ -0,0 +1,130 @@
+from datetime import datetime
+from typing import Any, Literal, Optional
+
+import click
+import click_spinner
+
+from shapely.geometry import mapping, MultiPolygon, Polygon, shape
+
+from mapchete.cli.options import opt_bounds, opt_debug
+from mapchete.io import fiona_open
+from mapchete.path import MPath
+from mapchete.types import Bounds
+
+from mapchete_eo.cli import options_arguments
+from mapchete_eo.io.products import Slice, products_to_slices
+from mapchete_eo.platforms.sentinel2.product import S2Product
+from mapchete_eo.platforms.sentinel2.source import Sentinel2Source
+from mapchete_eo.sort import TargetDateSort
+from mapchete_eo.types import TimeRange
+
+
+@click.command()
+@options_arguments.arg_dst_path
+@options_arguments.opt_start_time
+@options_arguments.opt_end_time
+@opt_bounds
+@options_arguments.opt_mgrs_tile
+@options_arguments.opt_source
+@click.option(
+    "--format",
+    type=click.Choice(["FlatGeobuf", "GeoJSON"]),
+    help="Format of output file.",
+)
+@click.option("--by-slices", is_flag=True, help="Merge product to slices.")
+@click.option(
+    "--add-index", is_flag=True, help="Add unique indexes to products/slices."
+)
+@opt_debug
+def s2_cat_results(
+    dst_path: MPath,
+    start_time: datetime,
+    end_time: datetime,
+    bounds: Optional[Bounds] = None,
+    mgrs_tile: Optional[str] = None,
+    source: Sentinel2Source = Sentinel2Source(collection="EarthSearch"),
+    format: Literal["FlatGeobuf", "GeoJSON"] = "FlatGeobuf",
+    by_slices: bool = False,
+    add_index: bool = False,
+    debug: bool = False,
+):
+    """Write a search result."""
+    if any([start_time is None, end_time is None]):  # pragma: no cover
+        raise click.ClickException("--start-time and --end-time are mandatory")
+    if all([bounds is None, mgrs_tile is None]):  # pragma: no cover
+        raise click.ClickException("--bounds or --mgrs-tile are required")
+    slice_property_key = "s2:datastrip_id"
+    with click_spinner.Spinner(disable=debug):
+        catalog = source.get_catalog()
+        slices = products_to_slices(
+            [
+                S2Product.from_stac_item(item)
+                for item in catalog.search(
+                    time=TimeRange(start=start_time, end=end_time),
+                    bounds=bounds,
+                    search_kwargs=dict(mgrs_tile=mgrs_tile),
+                )
+            ],
+            group_by_property=slice_property_key if by_slices else None,
+            sort=TargetDateSort(target_date=start_time),
+        )
+    if slices:
+        schema = get_schema(by_slices=by_slices, add_index=add_index)
+        with fiona_open(
+            dst_path, mode="w", schema=schema, crs="EPSG:4326", format=format
+        ) as dst:
+            for index, _slice in enumerate(slices, start=1):
+                # 2025-4 agreed to make outputs multipolygons
+                # Convert the _slice.__geom_interface__ to Multipolygon if not the case
+
+                # Ensure the result is always a MultiPolygon even if only single Polygon is returned
+                # Else split features should come here as MultiPolygons
+                slice_shape = shape(_slice.__geom_interface__)
+                if isinstance(slice_shape, Polygon):
+                    slice_multipolygon = mapping(MultiPolygon([slice_shape]))
+                else:
+                    slice_multipolygon = _slice.__geom_interface__
+
+                out_feature = {
+                    "geometry": slice_multipolygon,
+                    "properties": {
+                        key: get_value(_slice, key, index, slice_property_key)
+                        for key in schema["properties"].keys()
+                    },
+                }
+                dst.write(out_feature)
+    else:
+        click.echo("No results found.")
+
+
+def get_schema(
+    by_slices: bool, add_index: bool, geometry_type: str = "MultiPolygon"
+) -> dict:
+    if by_slices:
+        properties = {
+            "timestamp": "str",
+            "slice_id": "str",
+        }
+    else:
+        properties = {
+            "eo:cloud_cover": "float",
+            "timestamp": "str",
+            "slice_id": "str",
+            "product_id": "str",
+        }
+    if add_index:
+        properties.update(index="int")
+    return {"geometry": geometry_type, "properties": properties}
+
+
+def get_value(_slice: Slice, key: str, index: int, slice_property_key: str) -> Any:
+    if key == "index":
+        return index
+    elif key == "slice_id":
+        return _slice.get_property(slice_property_key)
+    elif key == "product_id":
+        return _slice.products[0].item.id
+    elif key == "timestamp":
+        return _slice.datetime
+    else:
+        return _slice.get_property(key)

mapchete_eo/cli/s2_find_broken_products.py

@@ -0,0 +1,77 @@
+from datetime import datetime
+from typing import List, Optional
+
+import click
+from mapchete.cli.options import opt_bounds, opt_debug
+from mapchete.path import MPath
+from mapchete.types import Bounds
+from tqdm import tqdm
+
+from mapchete_eo.cli import options_arguments
+from mapchete_eo.cli.s2_verify import verify_item
+from mapchete_eo.platforms.sentinel2.source import Sentinel2Source
+from mapchete_eo.product import add_to_blacklist, blacklist_products
+from mapchete_eo.types import TimeRange
+
+
+@click.command()
+@opt_bounds
+@options_arguments.opt_start_time
+@options_arguments.opt_end_time
+@options_arguments.opt_source
+@options_arguments.opt_assets
+@options_arguments.opt_blacklist
+@options_arguments.opt_thumbnail_dir
+@opt_debug
+def s2_find_broken_products(
+    start_time: datetime,
+    end_time: datetime,
+    bounds: Optional[Bounds] = None,
+    mgrs_tile: Optional[str] = None,
+    source: Sentinel2Source = Sentinel2Source(collection="EarthSearch"),
+    assets: List[str] = [],
+    asset_exists_check: bool = True,
+    blacklist: MPath = MPath("s3://eox-mhub-cache/blacklist.txt"),
+    thumbnail_dir: Optional[MPath] = None,
+    **__,
+):
+    """Find broken Sentinel-2 products."""
+    if any([start_time is None, end_time is None]):  # pragma: no cover
+        raise click.ClickException("--start-time and --end-time are mandatory")
+    if all([bounds is None, mgrs_tile is None]):  # pragma: no cover
+        raise click.ClickException("--bounds or --mgrs-tile are required")
+    catalog = source.get_catalog()
+    blacklisted_products = blacklist_products(blacklist)
+    for item in tqdm(
+        catalog.search(
+            time=TimeRange(start=start_time, end=end_time),
+            bounds=bounds,
+            search_kwargs=dict(mgrs_tile=mgrs_tile),
+        )
+    ):
+        report = verify_item(
+            item,
+            assets=assets,
+            asset_exists_check=asset_exists_check,
+            thumbnail_dir=thumbnail_dir,
+        )
+        for asset in report.missing_asset_entries:
+            tqdm.write(f"[ERROR] {report.item.id} has no asset named '{asset}")
+        for path in report.missing_assets:
+            tqdm.write(
+                f"[ERROR] {report.item.id} asset '{asset}' with path {str(path)} does not exist"
+            )
+        if report.color_artefacts:
+            tqdm.write(
+                f"[ERROR] {report.item.id} thumbnail ({report.item.assets['thumbnail'].href}) indicates that there are some color artefacts"
+            )
+        if report.product_broken():
+            if report.item.get_self_href() in blacklisted_products:
+                tqdm.write(f"product {report.item.id} already in blacklist")
+            elif click.confirm(
+                f"should product {report.item.id} be added to the blacklist ({str(blacklist)})"
+            ):
+                add_to_blacklist(
+                    report.item.get_self_href(),
+                    blacklist=blacklist,
+                )

mapchete_eo/cli/s2_jp2_static_catalog.py

@@ -0,0 +1,166 @@
+from __future__ import annotations
+
+import logging
+from collections import defaultdict
+from dataclasses import dataclass, field
+from datetime import datetime
+from types import TracebackType
+from typing import Optional, Type
+
+import click
+import tqdm
+from fiona.crs import CRS
+from mapchete.cli.options import opt_bounds, opt_debug
+from mapchete.io.vector import fiona_open
+from mapchete.path import MPath
+from mapchete.types import Bounds
+from pystac import Item
+from shapely import prepare
+
+from mapchete_eo.cli import options_arguments
+from mapchete_eo.io.items import item_fix_footprint
+from mapchete_eo.search.s2_mgrs import InvalidMGRSSquare, S2Tile
+from mapchete_eo.time import day_range
+
+logger = logging.getLogger(__name__)
+
+
+DEFAULT_SCHEMA = {
+    "geometry": ("Polygon", "MultiPolygon"),
+    "properties": {"id": "str", "path": "str"},
+}
+
+
+@dataclass
+class VectorDataSource:
+    path: MPath
+    schema: Optional[dict] = None
+    driver: Optional[str] = "FlatGeobuf"
+    crs: Optional[CRS] = CRS.from_epsg(4326)
+    features: dict = field(default_factory=dict)
+
+    def write(self, feature_id: str, feature: dict):
+        if feature_id not in self.features:
+            self.features[feature_id] = dict(
+                id=feature_id,
+                geometry=feature["geometry"],
+                properties=dict(feature["properties"], id=feature_id),
+            )
+
+    def __enter__(self) -> VectorDataSource:
+        if self.path.exists():
+            logger.debug("read existing data from %s", str(self.path))
+            with fiona_open(self.path) as src:
+                self.features = {
+                    feature.properties["id"]: dict(
+                        geometry=dict(feature.geometry),
+                        properties=dict(feature.properties),
+                    )
+                    for feature in src
+                }
+
+        return self
+
+    def __exit__(
+        self,
+        exc_type: Optional[Type[Exception]] = None,
+        exc: Optional[Exception] = None,
+        exc_tb: Optional[TracebackType] = None,
+    ):
+        if not exc_type:
+            logger.debug(f"remove {self.path} if exists ...")
+            self.path.rm(ignore_errors=True)
+            logger.debug(f"writing to {self.path} ...")
+            with fiona_open(
+                self.path,
+                mode="w",
+                driver=self.driver,
+                schema=self.schema or DEFAULT_SCHEMA,
+                crs=self.crs,
+            ) as dst:
+                dst.writerecords(list(self.features.values()))
+
+
+@click.command()
+@options_arguments.arg_dst_path
+@options_arguments.opt_start_time
+@options_arguments.opt_end_time
+@opt_bounds
+@click.option(
+    "--basepath", type=click.Path(path_type=MPath), default="s3://sentinel-s2-l2a-stac/"
+)
+@opt_debug
+def s2_jp2_static_catalog(
+    dst_path: MPath,
+    start_time: datetime,
+    end_time: datetime,
+    bounds: Optional[Bounds] = None,
+    basepath: MPath = MPath("s3://sentinel-s2-l2a-stac/"),
+    **_,
+):
+    """
+    Create a queriable set of static files for AWS_JP2 STAC items.
+
+    - one master file linking spatially to all S2Tile subfiles: index.fgb
+    - each entry links to a specific S2Tile file, e.g. tiles/53NQJ.fgb
+    - each S2Tile file contains for each STAC item one entry with geometry and href
+    """
+    bounds = bounds or Bounds(-180, -90, 180, 90)
+    aoi = bounds.latlon_geometry()
+    prepare(aoi)
+    items_per_tile = defaultdict(list)
+    for day in day_range(start_date=start_time, end_date=end_time):
+        day_path = basepath / day.strftime("%Y/%m/%d")
+        click.echo(f"looking into {day_path} ...")
+        try:
+            paths = day_path.ls()
+        except FileNotFoundError:
+            continue
+        click.echo(f"found {len(paths)} items")
+        for json_path in paths:
+            tile_id = json_path.without_suffix().name.split("_")[-1]
+            items_per_tile[tile_id].append(json_path)
+
+    index_path = dst_path / "index.fgb"
+    s2tile_directory = MPath("s2tiles")
+    with VectorDataSource(path=index_path) as index:
+        for tile_id, json_paths in tqdm.tqdm(items_per_tile.items()):
+            try:
+                s2tile = S2Tile.from_tile_id(tile_id)
+            except InvalidMGRSSquare as exc:
+                logger.debug("omitting S2Tile because of %s", str(exc))
+                continue
+            if aoi.intersects(s2tile.latlon_geometry):
+                tqdm.tqdm.write(f"adding {s2tile.tile_id} ...")
+                relative_tile_index_path = s2tile_directory / f"{s2tile.tile_id}.fgb"
+                with VectorDataSource(
+                    path=dst_path / relative_tile_index_path,
+                    schema={
+                        "geometry": ("Polygon", "MultiPolygon"),
+                        "properties": {"id": "str", "path": "str", "datetime": "str"},
+                    },
+                ) as tile_index:
+                    for json_path in json_paths:
+                        item = item_fix_footprint(Item.from_file(json_path))
+                        if item.geometry:
+                            tile_index.write(
+                                item.id,
+                                dict(
+                                    geometry=dict(item.geometry),
+                                    properties=dict(
+                                        path=str(json_path), datetime=str(item.datetime)
+                                    ),
+                                ),
+                            )
+
+                index.write(
+                    feature_id=s2tile.tile_id,
+                    feature=dict(
+                        geometry=s2tile.latlon_geometry,
+                        properties=dict(
+                            path=str(relative_tile_index_path),
+                        ),
+                    ),
+                )
+            else:
+                tqdm.tqdm.write(f"{s2tile.tile_id} not within bounds")

mapchete_eo/cli/s2_mask.py

@@ -0,0 +1,71 @@
+import click
+import numpy as np
+import pystac
+from mapchete.cli.options import opt_debug
+from mapchete.io import rasterio_open
+
+from mapchete_eo.cli import options_arguments
+from mapchete_eo.platforms.sentinel2.config import MaskConfig, SceneClassification
+from mapchete_eo.platforms.sentinel2.product import S2Product
+
+
+@click.command()
+@options_arguments.arg_stac_item
+@options_arguments.arg_dst_path
+@options_arguments.opt_resolution
+@options_arguments.opt_rio_profile
+@options_arguments.opt_mask_footprint
+@options_arguments.opt_mask_clouds
+@options_arguments.opt_mask_snow_ice
+@options_arguments.opt_mask_cloud_probability_threshold
+@options_arguments.opt_mask_snow_probability_threshold
+@options_arguments.opt_mask_scl_classes
+@opt_debug
+def s2_mask(
+    stac_item,
+    dst_path,
+    resolution=None,
+    rio_profile=None,
+    mask_footprint=False,
+    mask_clouds=False,
+    mask_snow_ice=False,
+    mask_cloud_probability_threshold=100,
+    mask_snow_probability_threshold=100,
+    mask_scl_classes=None,
+    **_,
+):
+    """Generate mask for Sentinel-2 product from metadata."""
+    item = pystac.Item.from_file(stac_item)
+    product = S2Product.from_stac_item(item)
+    grid = product.metadata.grid(resolution)
+    click.echo(product)
+    mask_config = MaskConfig(
+        footprint=mask_footprint,
+        l1c_clouds=mask_clouds,
+        snow_ice=mask_snow_ice,
+        cloud_probability=mask_cloud_probability_threshold != 100,
+        cloud_probability_threshold=mask_cloud_probability_threshold,
+        snow_probability=mask_snow_probability_threshold != 100,
+        snow_probability_threshold=mask_snow_probability_threshold,
+        scl=bool(mask_scl_classes),
+        scl_classes=[
+            SceneClassification[scene_class] for scene_class in mask_scl_classes
+        ]
+        if bool(mask_scl_classes)
+        else None,
+    )
+    mask = product.get_mask(mask_config=mask_config).data
+    rgb = np.stack([mask * 255, mask, mask])
+    with rasterio_open(
+        dst_path,
+        mode="w",
+        crs=grid.crs,
+        transform=grid.transform,
+        width=grid.width,
+        height=grid.height,
+        dtype=np.uint8,
+        count=3,
+        nodata=0,
+        **rio_profile,
+    ) as dst:
+        dst.write(rgb)

mapchete_eo/cli/s2_mgrs.py

@@ -0,0 +1,45 @@
+import click
+from mapchete.cli.options import opt_bounds, opt_debug
+from mapchete.io import fiona_open
+from mapchete.path import MPath
+from mapchete.types import Bounds
+from shapely.geometry import mapping
+
+from mapchete_eo.cli import options_arguments
+from mapchete_eo.search.s2_mgrs import s2_tiles_from_bounds
+
+
+@click.command()
+@options_arguments.arg_dst_path
+@opt_bounds
+@opt_debug
+def s2_mgrs(
+    dst_path: MPath,
+    bounds: Bounds,
+    **_,
+):
+    """Save Sentinel-2 tile grid as FlatGeobuf."""
+    schema = dict(
+        geometry="Polygon",
+        properties=dict(
+            utm_zone="str",
+            latitude_band="str",
+            grid_square="str",
+            tile_id="str",
+        ),
+    )
+    with fiona_open(
+        dst_path, "w", crs="EPSG:4326", driver="FlatGeobuf", schema=schema
+    ) as dst:
+        for tile in s2_tiles_from_bounds(*bounds):
+            dst.write(
+                dict(
+                    geometry=mapping(tile.latlon_geometry),
+                    properties=dict(
+                        utm_zone=tile.utm_zone,
+                        latitude_band=tile.latitude_band,
+                        grid_square=tile.grid_square,
+                        tile_id=tile.tile_id,
+                    ),
+                )
+            )

mapchete_eo/cli/s2_rgb.py

@@ -0,0 +1,114 @@
+from typing import List
+
+import click
+import numpy as np
+from numpy.typing import DTypeLike
+import pystac
+from mapchete.cli.options import opt_debug
+from mapchete.io import rasterio_open
+from mapchete.path import MPath
+from mapchete import Timer
+
+from mapchete_eo.cli import options_arguments
+from mapchete_eo.image_operations import linear_normalization
+from mapchete_eo.platforms.sentinel2.config import (
+    BRDFConfig,
+    MaskConfig,
+    SceneClassification,
+)
+from mapchete_eo.platforms.sentinel2.product import S2Product
+from mapchete_eo.platforms.sentinel2.types import Resolution
+
+
+@click.command()
+@options_arguments.arg_stac_item
+@options_arguments.opt_dst_path
+@options_arguments.opt_assets_rgb
+@options_arguments.opt_resolution
+@options_arguments.opt_rio_profile
+@options_arguments.opt_mask_footprint
+@options_arguments.opt_mask_snow_ice
+@options_arguments.opt_mask_cloud_probability_threshold
+@options_arguments.opt_mask_snow_probability_threshold
+@options_arguments.opt_mask_scl_classes
+@options_arguments.opt_brdf_model
+@options_arguments.opt_brdf_weight
+@options_arguments.opt_brdf_log10
+@options_arguments.opt_brdf_detector_iter
+@options_arguments.opt_out_dtype
+@opt_debug
+def s2_rgb(
+    stac_item: MPath,
+    dst_path: MPath,
+    assets: List[str] = ["red", "green", "blue"],
+    resolution: Resolution = Resolution["120m"],
+    rio_profile=None,
+    mask_footprint=False,
+    mask_snow_ice=False,
+    mask_cloud_probability_threshold=100,
+    mask_snow_probability_threshold=100,
+    mask_scl_classes=None,
+    brdf_model=None,
+    brdf_weight: float = 1.0,
+    brdf_log10: bool = False,
+    brdf_detector_iter: bool = False,
+    out_dtype: DTypeLike = "uint8",
+    **_,
+):
+    """Generate 8bit RGB image from Sentinel-2 product."""
+    out_dtype = np.dtype(out_dtype)
+
+    if not dst_path.suffix:
+        dst_path = dst_path / stac_item.without_suffix().name + ".tif"
+    if resolution == Resolution.original:
+        resolution = Resolution["10m"]
+    product = S2Product.from_stac_item(pystac.Item.from_file(stac_item))
+    grid = product.metadata.grid(resolution)
+    click.echo(
+        f"writing {stac_item} assets {', '.join(assets)} to {dst_path} in {resolution.name} resolution"
+    )
+    with Timer() as t:
+        mask_config = MaskConfig(
+            footprint=mask_footprint,
+            snow_ice=mask_snow_ice,
+            cloud_probability_threshold=mask_cloud_probability_threshold,
+            snow_probability_threshold=mask_snow_probability_threshold,
+            scl_classes=(
+                [SceneClassification[scene_class] for scene_class in mask_scl_classes]
+                if bool(mask_scl_classes)
+                else None
+            ),
+        )
+        rgb = product.read_np_array(
+            assets=assets,
+            grid=grid,
+            mask_config=mask_config,
+            brdf_config=BRDFConfig(
+                bands=assets,
+                model=brdf_model,
+                correction_weight=brdf_weight,
+                log10_bands_scale=brdf_log10,
+                per_detector_correction=brdf_detector_iter,
+            )
+            if brdf_model
+            else None,
+        )
+        with rasterio_open(
+            dst_path,
+            mode="w",
+            crs=grid.crs,
+            transform=grid.transform,
+            width=grid.width,
+            height=grid.height,
+            dtype=out_dtype,
+            count=len(assets),
+            nodata=0,
+            **rio_profile,
+        ) as dst:
+            if out_dtype == np.uint8:
+                dst.write(linear_normalization(rgb, out_min=1))
+            else:
+                dst.write(rgb)
+    click.echo(
+        f"{stac_item} assets {', '.join(assets)} to {dst_path} in {resolution.name} written in {t}"
+    )