mapchete-eo 2025.10.1__py2.py3-none-any.whl → 2025.11.0__py2.py3-none-any.whl

mapchete_eo/__init__.py

@@ -1 +1 @@
-__version__ = "2025.10.1"
+__version__ = "2025.11.0"

mapchete_eo/base.py

@@ -1,8 +1,9 @@
 from __future__ import annotations
 
+import warnings
 import logging
 from functools import cached_property
-from typing import Any, Callable, List, Optional, Type, Union
+from typing import Any, Callable, List, Optional, Sequence, Type, Union, Dict, Generator
 
 import croniter
 from mapchete import Bounds
@@ -17,13 +18,13 @@ from mapchete.io.vector import IndexedFeatures
 from mapchete.path import MPath
 from mapchete.tile import BufferedTile
 from mapchete.types import MPathLike, NodataVal, NodataVals
-from pydantic import BaseModel
+from pydantic import BaseModel, model_validator
+from pystac import Item
 from rasterio.enums import Resampling
 from rasterio.features import geometry_mask
 from shapely.geometry import mapping
 from shapely.geometry.base import BaseGeometry
 
-from mapchete_eo.archives.base import Archive
 from mapchete_eo.exceptions import CorruptedProductMetadata, PreprocessingNotFinished
 from mapchete_eo.io import (
     products_to_np_array,
@@ -31,9 +32,9 @@ from mapchete_eo.io import (
     read_levelled_cube_to_np_array,
     read_levelled_cube_to_xarray,
 )
+from mapchete_eo.source import Source
 from mapchete_eo.product import EOProduct
 from mapchete_eo.protocols import EOProductProtocol
-from mapchete_eo.search.stac_static import STACStaticCatalog
 from mapchete_eo.settings import mapchete_eo_settings
 from mapchete_eo.sort import SortMethodConfig, TargetDateSort
 from mapchete_eo.time import to_datetime
@@ -44,13 +45,39 @@ logger = logging.getLogger(__name__)
 
 class BaseDriverConfig(BaseModel):
     format: str
-    time: Union[TimeRange, List[TimeRange]]
+    source: Sequence[Source]
+    time: Optional[Union[TimeRange, List[TimeRange]]] = None
     cat_baseurl: Optional[str] = None
     cache: Optional[Any] = None
     footprint_buffer: float = 0
     area: Optional[Union[MPathLike, dict, type[BaseGeometry]]] = None
     preprocessing_tasks: bool = False
-    archive: Optional[Type[Archive]] = None
+    search_kwargs: Optional[Dict[str, Any]] = None
+
+    @model_validator(mode="before")
+    def to_list(cls, values: Dict[str, Any]) -> Dict[str, Any]:
+        """Expands source to list."""
+        for field in ["source"]:
+            value = values.get(field)
+            if value is not None and not isinstance(value, list):
+                values[field] = [value]
+        return values
+
+    @model_validator(mode="before")
+    def deprecate_cat_baseurl(cls, values: Dict[str, Any]) -> Dict[str, Any]:
+        cat_baseurl = values.get("cat_baseurl")
+        if cat_baseurl:  # pragma: no cover
+            warnings.warn(
+                "'cat_baseurl' will be deprecated soon. Please use 'catalog_type=static' in the source.",
+                category=DeprecationWarning,
+                stacklevel=2,
+            )
+            if values.get("source", []):
+                raise ValueError(
+                    "deprecated cat_baseurl field found alongside sources."
+                )
+            values["source"] = [dict(collection=cat_baseurl, catalog_type="static")]
+        return values
 
 
 class EODataCube(base.InputTile):
@@ -63,7 +90,7 @@ class EODataCube(base.InputTile):
 
     tile: BufferedTile
     eo_bands: dict
-    time: List[TimeRange]
+    time: Optional[List[TimeRange]]
     area: BaseGeometry
     area_pixelbuffer: int = 0
 
@@ -72,7 +99,7 @@ class EODataCube(base.InputTile):
         tile: BufferedTile,
         products: Optional[List[EOProductProtocol]],
         eo_bands: dict,
-        time: List[TimeRange],
+        time: Optional[List[TimeRange]] = None,
         input_key: Optional[str] = None,
         area: Optional[BaseGeometry] = None,
         **kwargs,
@@ -314,27 +341,25 @@ class EODataCube(base.InputTile):
         """
         Return a filtered list of input products.
         """
-        if any([start_time, end_time, timestamps]):
+        if any([start_time, end_time, timestamps]):  # pragma: no cover
             raise NotImplementedError("time subsets are not yet implemented")
 
         if time_pattern:
             # filter products by time pattern
-            tz = tzutc()
-            coord_time = [
-                t.replace(tzinfo=tz)
-                for t in croniter.croniter_range(
-                    to_datetime(self.start_time),
-                    to_datetime(self.end_time),
-                    time_pattern,
-                )
-            ]
             return [
                 product
                 for product in self.products
-                if product.item.datetime in coord_time
+                if product.item.datetime
+                in [
+                    t.replace(tzinfo=tzutc())
+                    for t in croniter.croniter_range(
+                        to_datetime(self.start_time),
+                        to_datetime(self.end_time),
+                        time_pattern,
+                    )
+                ]
             ]
-        else:
-            return self.products
+        return self.products
 
     def is_empty(self) -> bool:  # pragma: no cover
         """
@@ -358,16 +383,16 @@ class EODataCube(base.InputTile):
             nodatavals = self.default_read_nodataval
         merge_products_by = merge_products_by or self.default_read_merge_products_by
         merge_method = merge_method or self.default_read_merge_method
-        if resampling is None:
-            resampling = self.default_read_resampling
-        else:
-            resampling = (
-                resampling
-                if isinstance(resampling, Resampling)
-                else Resampling[resampling]
-            )
         return dict(
-            resampling=resampling,
+            resampling=(
+                self.default_read_resampling
+                if resampling is None
+                else (
+                    resampling
+                    if isinstance(resampling, Resampling)
+                    else Resampling[resampling]
+                )
+            ),
             nodatavals=nodatavals,
             merge_products_by=merge_products_by,
             merge_method=merge_method,
@@ -401,8 +426,7 @@ class InputData(base.InputData):
     default_preprocessing_task: Callable = staticmethod(EOProduct.from_stac_item)
     driver_config_model: Type[BaseDriverConfig] = BaseDriverConfig
     params: BaseDriverConfig
-    archive: Archive
-    time: Union[TimeRange, List[TimeRange]]
+    time: Optional[Union[TimeRange, List[TimeRange]]]
     area: BaseGeometry
     _products: Optional[IndexedFeatures] = None
 
@@ -421,6 +445,8 @@ class InputData(base.InputData):
         self.standalone = standalone
 
         self.params = self.driver_config_model(**input_params["abstract"])
+        self.conf_dir = input_params.get("conf_dir")
+
         # we have to make sure, the cache path is absolute
         # not quite fond of this solution
         if self.params.cache:
@@ -429,14 +455,18 @@ class InputData(base.InputData):
             ).absolute_path(base_dir=input_params.get("conf_dir"))
         self.area = self._init_area(input_params)
         self.time = self.params.time
-        if self.readonly:  # pragma: no cover
-            return
 
-        self.set_archive(base_dir=input_params["conf_dir"])
+        self.eo_bands = [
+            eo_band
+            for source in self.params.source
+            for eo_band in source.eo_bands(base_dir=self.conf_dir)
+        ]
 
+        if self.readonly:  # pragma: no cover
+            return
         # don't use preprocessing tasks for Sentinel-2 products:
         if self.params.preprocessing_tasks or self.params.cache is not None:
-            for item in self.archive.items():
+            for item in self.source_items():
                 self.add_preprocessing_task(
                     self.default_preprocessing_task,
                     fargs=(item,),
@@ -455,7 +485,7 @@ class InputData(base.InputData):
                     self.default_preprocessing_task(
                         item, cache_config=self.params.cache, cache_all=True
                     )
-                    for item in self.archive.items()
+                    for item in self.source_items()
                 ]
             )
 
@@ -467,7 +497,7 @@ class InputData(base.InputData):
             configured_area, configured_area_crs = guess_geometry(
                 self.params.area,
                 bounds=Bounds.from_inp(
-                    input_params.get("delimiters", {}).get("bounds"),
+                    input_params.get("delimiters", {}).get("effective_bounds"),
                     crs=getattr(input_params.get("pyramid"), "crs"),
                 ),
                 raise_if_empty=False,
@@ -481,20 +511,30 @@ class InputData(base.InputData):
             )
         return process_area
 
-    def set_archive(self, base_dir: MPath):
-        # this only works with some static archive:
-        if self.params.cat_baseurl:
-            self.archive = Archive(
-                catalog=STACStaticCatalog(
-                    baseurl=MPath(self.params.cat_baseurl).absolute_path(
-                        base_dir=base_dir
-                    ),
-                ),
-                area=self.bbox(mapchete_eo_settings.default_catalog_crs),
-                time=self.time,
+    def source_items(self) -> Generator[Item, None, None]:
+        already_returned = set()
+        for source in self.params.source:
+            area = reproject_geometry(
+                self.area,
+                src_crs=self.crs,
+                dst_crs=source.catalog_crs,
             )
-        else:
-            raise NotImplementedError()
+            if area.is_empty:
+                continue
+            for item in source.search(
+                time=self.time,
+                area=area,
+                base_dir=self.conf_dir,
+            ):
+                # if item was already found in previous source, skip
+                if item.id in already_returned:
+                    continue
+
+                # if item is new, add to list and yield
+                already_returned.add(item.id)
+                item.properties["mapchete_eo:source"] = source
+                yield item
+        logger.debug("returned set of %s items", len(already_returned))
 
     def bbox(self, out_crs: Optional[str] = None) -> BaseGeometry:
         """Return data bounding box."""
@@ -517,7 +557,7 @@ class InputData(base.InputData):
             return self._products
 
         # TODO: copied it from mapchete_satellite, not yet sure which use case this is
-        elif self.standalone:
+        elif self.standalone:  # pragma: no cover
             raise NotImplementedError()
 
         # if preprocessing tasks are ready, index them for further use
@@ -525,7 +565,7 @@ class InputData(base.InputData):
             return IndexedFeatures(
                 [
                     self.get_preprocessing_task_result(item.id)
-                    for item in self.archive.items()
+                    for item in self.source_items()
                     if not isinstance(item, CorruptedProductMetadata)
                 ],
                 crs=self.crs,
@@ -557,7 +597,7 @@ class InputData(base.InputData):
         return self.input_tile_cls(
             tile,
             products=tile_products,
-            eo_bands=self.archive.catalog.eo_bands,
+            eo_bands=self.eo_bands,
             time=self.time,
             # passing on the input key is essential so dependent preprocessing tasks can be found!
             input_key=self.input_key,

mapchete_eo/cli/options_arguments.py

@@ -6,8 +6,8 @@ from mapchete.path import MPath
 
 from mapchete_eo.platforms.sentinel2.brdf.models import BRDFModels
 from mapchete_eo.io.profiles import rio_profiles
-from mapchete_eo.platforms.sentinel2.archives import KnownArchives
 from mapchete_eo.platforms.sentinel2.config import SceneClassification
+from mapchete_eo.platforms.sentinel2.source import Sentinel2Source
 from mapchete_eo.platforms.sentinel2.types import L2ABand, Resolution
 from mapchete_eo.time import to_datetime
 
@@ -58,16 +58,15 @@ def _str_to_l2a_bands(_, __, value):
         return [L2ABand[v] for v in value.split(",")]
 
 
-def _archive_name_to_archive_cls(_, __, value):
+def _str_to_datetime(_, param, value):
     if value:
-        return KnownArchives[value]
+        return to_datetime(value)
+    raise ValueError(f"--{param.name} is mandatory")
 
 
-def _str_to_datetime(_, param, value):
+def _str_to_source(_, __, value):
     if value:
-        return to_datetime(value)
-    else:
-        raise ValueError(f"--{param.name} is mandatory")
+        return Sentinel2Source(collection=value)
 
 
 arg_stac_item = click.argument("stac-item", type=click.Path(path_type=MPath))
@@ -167,27 +166,12 @@ opt_start_time = click.option(
 opt_end_time = click.option(
     "--end-time", type=click.STRING, callback=_str_to_datetime, help="End time"
 )
-opt_archive = click.option(
-    "--archive",
-    type=click.Choice([archive.name for archive in KnownArchives]),
-    default="S2AWS_COG",
-    help="Archive to read from.",
-    callback=_archive_name_to_archive_cls,
-)
-opt_collection = click.option(
-    "--collection",
+opt_source = click.option(
+    "--source",
     type=click.STRING,
-    help="Data collection to be queried.",
-)
-opt_endpoint = click.option(
-    "--endpoint",
-    type=click.STRING,
-    help="Search endpoint.",
-)
-opt_catalog_json = click.option(
-    "--catalog-json",
-    type=click.Path(path_type=MPath),
-    help="JSON file for a static catalog.",
+    default="EarthSearch",
+    callback=_str_to_source,
+    help="Data source to be queried.",
 )
 opt_name = click.option("--name", type=click.STRING, help="Static catalog name.")
 opt_description = click.option(

mapchete_eo/cli/s2_brdf.py

@@ -11,7 +11,7 @@ from mapchete_eo.io.profiles import COGDeflateProfile
 from mapchete_eo.platforms.sentinel2.brdf.config import BRDFModels
 from mapchete_eo.platforms.sentinel2.config import BRDFConfig
 from mapchete_eo.platforms.sentinel2.product import S2Product
-from mapchete_eo.platforms.sentinel2.metadata_parser import Resolution
+from mapchete_eo.platforms.sentinel2.metadata_parser.s2metadata import Resolution
 from mapchete_eo.platforms.sentinel2.types import L2ABand
 
 

mapchete_eo/cli/s2_cat_results.py

@@ -12,10 +12,9 @@ from mapchete.path import MPath
 from mapchete.types import Bounds
 
 from mapchete_eo.cli import options_arguments
-from mapchete_eo.cli.static_catalog import get_catalog
 from mapchete_eo.io.products import Slice, products_to_slices
-from mapchete_eo.platforms.sentinel2.archives import KnownArchives
 from mapchete_eo.platforms.sentinel2.product import S2Product
+from mapchete_eo.platforms.sentinel2.source import Sentinel2Source
 from mapchete_eo.sort import TargetDateSort
 from mapchete_eo.types import TimeRange
 
@@ -26,10 +25,7 @@ from mapchete_eo.types import TimeRange
 @options_arguments.opt_end_time
 @opt_bounds
 @options_arguments.opt_mgrs_tile
-@options_arguments.opt_archive
-@options_arguments.opt_collection
-@options_arguments.opt_endpoint
-@options_arguments.opt_catalog_json
+@options_arguments.opt_source
 @click.option(
     "--format",
     type=click.Choice(["FlatGeobuf", "GeoJSON"]),
@@ -46,32 +42,20 @@ def s2_cat_results(
     end_time: datetime,
     bounds: Optional[Bounds] = None,
     mgrs_tile: Optional[str] = None,
-    archive: Optional[KnownArchives] = None,
-    collection: Optional[str] = None,
-    endpoint: Optional[str] = None,
-    catalog_json: Optional[MPath] = None,
+    source: Sentinel2Source = Sentinel2Source(collection="EarthSearch"),
     format: Literal["FlatGeobuf", "GeoJSON"] = "FlatGeobuf",
     by_slices: bool = False,
     add_index: bool = False,
     debug: bool = False,
 ):
     """Write a search result."""
-    if catalog_json and endpoint:  # pragma: no cover
-        raise click.ClickException(
-            "exactly one of --archive, --catalog-json or --endpoint has to be set."
-        )
     if any([start_time is None, end_time is None]):  # pragma: no cover
         raise click.ClickException("--start-time and --end-time are mandatory")
     if all([bounds is None, mgrs_tile is None]):  # pragma: no cover
         raise click.ClickException("--bounds or --mgrs-tile are required")
     slice_property_key = "s2:datastrip_id"
     with click_spinner.Spinner(disable=debug):
-        catalog = get_catalog(
-            catalog_json=catalog_json,
-            endpoint=endpoint,
-            known_archive=archive,
-            collection=collection,
-        )
+        catalog = source.get_catalog()
         slices = products_to_slices(
             [
                 S2Product.from_stac_item(item)

mapchete_eo/cli/s2_find_broken_products.py

@@ -9,8 +9,7 @@ from tqdm import tqdm
 
 from mapchete_eo.cli import options_arguments
 from mapchete_eo.cli.s2_verify import verify_item
-from mapchete_eo.cli.static_catalog import get_catalog
-from mapchete_eo.platforms.sentinel2.archives import KnownArchives
+from mapchete_eo.platforms.sentinel2.source import Sentinel2Source
 from mapchete_eo.product import add_to_blacklist, blacklist_products
 from mapchete_eo.types import TimeRange
 
@@ -19,10 +18,7 @@ from mapchete_eo.types import TimeRange
 @opt_bounds
 @options_arguments.opt_start_time
 @options_arguments.opt_end_time
-@options_arguments.opt_archive
-@options_arguments.opt_collection
-@options_arguments.opt_endpoint
-@options_arguments.opt_catalog_json
+@options_arguments.opt_source
 @options_arguments.opt_assets
 @options_arguments.opt_blacklist
 @options_arguments.opt_thumbnail_dir
@@ -32,10 +28,7 @@ def s2_find_broken_products(
     end_time: datetime,
     bounds: Optional[Bounds] = None,
     mgrs_tile: Optional[str] = None,
-    archive: Optional[KnownArchives] = None,
-    collection: Optional[str] = None,
-    endpoint: Optional[str] = None,
-    catalog_json: Optional[MPath] = None,
+    source: Sentinel2Source = Sentinel2Source(collection="EarthSearch"),
     assets: List[str] = [],
     asset_exists_check: bool = True,
     blacklist: MPath = MPath("s3://eox-mhub-cache/blacklist.txt"),
@@ -43,20 +36,11 @@ def s2_find_broken_products(
     **__,
 ):
     """Find broken Sentinel-2 products."""
-    if catalog_json and endpoint:  # pragma: no cover
-        raise click.ClickException(
-            "exactly one of --archive, --catalog-json or --endpoint has to be set."
-        )
     if any([start_time is None, end_time is None]):  # pragma: no cover
         raise click.ClickException("--start-time and --end-time are mandatory")
     if all([bounds is None, mgrs_tile is None]):  # pragma: no cover
         raise click.ClickException("--bounds or --mgrs-tile are required")
-    catalog = get_catalog(
-        catalog_json=catalog_json,
-        endpoint=endpoint,
-        known_archive=archive,
-        collection=collection,
-    )
+    catalog = source.get_catalog()
     blacklisted_products = blacklist_products(blacklist)
     for item in tqdm(
         catalog.search(

mapchete_eo/cli/s2_jp2_static_catalog.py

@@ -19,7 +19,7 @@ from shapely import prepare
 
 from mapchete_eo.cli import options_arguments
 from mapchete_eo.io.items import item_fix_footprint
-from mapchete_eo.search.s2_mgrs import InvalidMGRSSquare, S2Tile, bounds_to_geom
+from mapchete_eo.search.s2_mgrs import InvalidMGRSSquare, S2Tile
 from mapchete_eo.time import day_range
 
 logger = logging.getLogger(__name__)
@@ -106,7 +106,7 @@ def s2_jp2_static_catalog(
     - each S2Tile file contains for each STAC item one entry with geometry and href
     """
     bounds = bounds or Bounds(-180, -90, 180, 90)
-    aoi = bounds_to_geom(bounds)
+    aoi = bounds.latlon_geometry()
     prepare(aoi)
     items_per_tile = defaultdict(list)
     for day in day_range(start_date=start_time, end_date=end_time):

mapchete_eo/cli/static_catalog.py

@@ -9,10 +9,8 @@ from rasterio.profiles import Profile
 
 from mapchete_eo.cli import options_arguments
 from mapchete_eo.platforms.sentinel2 import S2Metadata
-from mapchete_eo.platforms.sentinel2.archives import KnownArchives
+from mapchete_eo.platforms.sentinel2.source import Sentinel2Source
 from mapchete_eo.platforms.sentinel2.types import Resolution
-from mapchete_eo.search import STACSearchCatalog, STACStaticCatalog
-from mapchete_eo.search.base import CatalogSearcher
 from mapchete_eo.types import TimeRange
 
 
@@ -22,10 +20,7 @@ from mapchete_eo.types import TimeRange
 @options_arguments.opt_mgrs_tile
 @options_arguments.opt_start_time
 @options_arguments.opt_end_time
-@options_arguments.opt_archive
-@options_arguments.opt_collection
-@options_arguments.opt_endpoint
-@options_arguments.opt_catalog_json
+@options_arguments.opt_source
 @options_arguments.opt_name
 @options_arguments.opt_description
 @options_arguments.opt_assets
@@ -40,10 +35,7 @@ def static_catalog(
     end_time: datetime,
     bounds: Optional[Bounds] = None,
     mgrs_tile: Optional[str] = None,
-    archive: Optional[KnownArchives] = None,
-    collection: Optional[str] = None,
-    endpoint: Optional[str] = None,
-    catalog_json: Optional[MPath] = None,
+    source: Sentinel2Source = Sentinel2Source(collection="EarthSearch"),
     name: Optional[str] = None,
     description: Optional[str] = None,
     assets: Optional[List[str]] = None,
@@ -54,20 +46,11 @@ def static_catalog(
     **__,
 ):
     """Write a static STAC catalog for selected area."""
-    if catalog_json and endpoint:  # pragma: no cover
-        raise click.ClickException(
-            "exactly one of --archive, --catalog-json or --endpoint has to be set."
-        )
     if any([start_time is None, end_time is None]):  # pragma: no cover
         raise click.ClickException("--start-time and --end-time are mandatory")
     if all([bounds is None, mgrs_tile is None]):  # pragma: no cover
         raise click.ClickException("--bounds or --mgrs-tile are required")
-    catalog = get_catalog(
-        catalog_json=catalog_json,
-        endpoint=endpoint,
-        known_archive=archive,
-        collection=collection,
-    )
+    catalog = source.get_catalog()
     if hasattr(catalog, "write_static_catalog"):
         with options_arguments.TqdmUpTo(
             unit="products", unit_scale=True, miniters=1, disable=opt_debug
@@ -97,27 +80,3 @@ def static_catalog(
         raise AttributeError(
             f"catalog {catalog} does not support writing a static version"
         )
-
-
-def get_catalog(
-    catalog_json: Optional[MPath],
-    endpoint: Optional[MPath],
-    known_archive: Optional[KnownArchives] = None,
-    collection: Optional[str] = None,
-) -> CatalogSearcher:
-    if catalog_json:
-        return STACStaticCatalog(
-            baseurl=catalog_json,
-        )
-    elif endpoint:
-        if collection:
-            return STACSearchCatalog(
-                endpoint=endpoint,
-                collections=[collection],
-            )
-        else:
-            raise ValueError("collection must be provided")
-    elif known_archive:
-        return known_archive.value.catalog
-    else:
-        raise TypeError("cannot determine catalog")

mapchete_eo/eostac.py

@@ -5,7 +5,7 @@ Driver class for EOSTAC static STAC catalogs.
 from mapchete_eo import base
 
 METADATA: dict = {
-    "driver_name": "EOSTAC_DEV",
+    "driver_name": "EOSTAC",
     "data_type": None,
     "mode": "r",
     "file_extensions": [],

mapchete_eo/io/assets.py

@@ -82,7 +82,7 @@ def asset_to_np_array(
     path = asset_mpath(item, asset)
 
     # find out asset details if raster:bands is available
-    stac_raster_bands = STACRasterBandProperties.from_asset(
+    band_properties = STACRasterBandProperties.from_asset(
         item.assets[asset], nodataval=nodataval
     )
 
@@ -92,23 +92,23 @@ def asset_to_np_array(
         indexes=indexes,
         grid=grid,
         resampling=resampling.name,
-        dst_nodata=stac_raster_bands.nodata,
+        dst_nodata=band_properties.nodata,
     ).data
 
-    if apply_offset and stac_raster_bands.offset:
-        data_type = stac_raster_bands.data_type or data.dtype
+    if apply_offset and band_properties.offset:
+        data_type = band_properties.data_type or data.dtype
 
         # determine value range for the target data_type
         clip_min, clip_max = dtype_ranges[str(data_type)]
 
         # increase minimum clip value to avoid collission with nodata value
-        if clip_min == stac_raster_bands.nodata:
+        if clip_min == band_properties.nodata:
             clip_min += 1
 
         data[:] = (
             (
-                ((data * stac_raster_bands.scale) + stac_raster_bands.offset)
-                / stac_raster_bands.scale
+                ((data * band_properties.scale) + band_properties.offset)
+                / band_properties.scale
             )
             .round()
             .clip(clip_min, clip_max)