mapchete-eo 2025.10.1__py2.py3-none-any.whl → 2026.1.0__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mapchete_eo/__init__.py +1 -1
- mapchete_eo/base.py +94 -54
- mapchete_eo/cli/options_arguments.py +11 -27
- mapchete_eo/cli/s2_brdf.py +1 -1
- mapchete_eo/cli/s2_cat_results.py +4 -20
- mapchete_eo/cli/s2_find_broken_products.py +4 -20
- mapchete_eo/cli/s2_jp2_static_catalog.py +2 -2
- mapchete_eo/cli/static_catalog.py +4 -45
- mapchete_eo/eostac.py +1 -1
- mapchete_eo/io/assets.py +20 -16
- mapchete_eo/io/items.py +36 -23
- mapchete_eo/io/path.py +19 -8
- mapchete_eo/io/products.py +22 -24
- mapchete_eo/platforms/sentinel2/__init__.py +1 -1
- mapchete_eo/platforms/sentinel2/_mapper_registry.py +89 -0
- mapchete_eo/platforms/sentinel2/brdf/correction.py +1 -1
- mapchete_eo/platforms/sentinel2/brdf/hls.py +1 -1
- mapchete_eo/platforms/sentinel2/brdf/models.py +1 -1
- mapchete_eo/platforms/sentinel2/brdf/protocols.py +1 -1
- mapchete_eo/platforms/sentinel2/brdf/ross_thick.py +1 -1
- mapchete_eo/platforms/sentinel2/brdf/sun_angle_arrays.py +1 -1
- mapchete_eo/platforms/sentinel2/config.py +73 -13
- mapchete_eo/platforms/sentinel2/driver.py +0 -39
- mapchete_eo/platforms/sentinel2/metadata_parser/__init__.py +6 -0
- mapchete_eo/platforms/sentinel2/{path_mappers → metadata_parser}/base.py +1 -1
- mapchete_eo/platforms/sentinel2/{path_mappers/metadata_xml.py → metadata_parser/default_path_mapper.py} +2 -2
- mapchete_eo/platforms/sentinel2/metadata_parser/models.py +78 -0
- mapchete_eo/platforms/sentinel2/{metadata_parser.py → metadata_parser/s2metadata.py} +51 -144
- mapchete_eo/platforms/sentinel2/preconfigured_sources/__init__.py +57 -0
- mapchete_eo/platforms/sentinel2/preconfigured_sources/guessers.py +108 -0
- mapchete_eo/platforms/sentinel2/preconfigured_sources/item_mappers.py +171 -0
- mapchete_eo/platforms/sentinel2/preconfigured_sources/metadata_xml_mappers.py +217 -0
- mapchete_eo/platforms/sentinel2/preprocessing_tasks.py +22 -1
- mapchete_eo/platforms/sentinel2/processing_baseline.py +3 -0
- mapchete_eo/platforms/sentinel2/product.py +83 -18
- mapchete_eo/platforms/sentinel2/source.py +114 -0
- mapchete_eo/platforms/sentinel2/types.py +5 -0
- mapchete_eo/product.py +14 -8
- mapchete_eo/protocols.py +5 -0
- mapchete_eo/search/__init__.py +3 -3
- mapchete_eo/search/base.py +127 -99
- mapchete_eo/search/config.py +75 -4
- mapchete_eo/search/s2_mgrs.py +8 -9
- mapchete_eo/search/stac_search.py +99 -97
- mapchete_eo/search/stac_static.py +46 -102
- mapchete_eo/search/utm_search.py +54 -62
- mapchete_eo/settings.py +1 -0
- mapchete_eo/sort.py +4 -6
- mapchete_eo/source.py +107 -0
- {mapchete_eo-2025.10.1.dist-info → mapchete_eo-2026.1.0.dist-info}/METADATA +4 -3
- mapchete_eo-2026.1.0.dist-info/RECORD +89 -0
- {mapchete_eo-2025.10.1.dist-info → mapchete_eo-2026.1.0.dist-info}/WHEEL +1 -1
- {mapchete_eo-2025.10.1.dist-info → mapchete_eo-2026.1.0.dist-info}/entry_points.txt +1 -1
- {mapchete_eo-2025.10.1.dist-info → mapchete_eo-2026.1.0.dist-info}/licenses/LICENSE +1 -1
- mapchete_eo/archives/__init__.py +0 -0
- mapchete_eo/archives/base.py +0 -65
- mapchete_eo/geometry.py +0 -271
- mapchete_eo/known_catalogs.py +0 -42
- mapchete_eo/platforms/sentinel2/archives.py +0 -190
- mapchete_eo/platforms/sentinel2/path_mappers/__init__.py +0 -29
- mapchete_eo/platforms/sentinel2/path_mappers/earthsearch.py +0 -34
- mapchete_eo/platforms/sentinel2/path_mappers/sinergise.py +0 -105
- mapchete_eo-2025.10.1.dist-info/RECORD +0 -88
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
import warnings
|
|
5
|
+
from typing import Dict
|
|
6
|
+
|
|
7
|
+
import numpy as np
|
|
8
|
+
import numpy.ma as ma
|
|
9
|
+
from pydantic import BaseModel
|
|
10
|
+
from mapchete.io.raster import ReferencedRaster
|
|
11
|
+
from rasterio.fill import fillnodata
|
|
12
|
+
|
|
13
|
+
from mapchete_eo.exceptions import CorruptedProductMetadata
|
|
14
|
+
from mapchete_eo.platforms.sentinel2.types import (
|
|
15
|
+
SunAngle,
|
|
16
|
+
ViewAngle,
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
logger = logging.getLogger(__name__)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class SunAngleData(BaseModel):
|
|
23
|
+
model_config = dict(arbitrary_types_allowed=True)
|
|
24
|
+
raster: ReferencedRaster
|
|
25
|
+
mean: float
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class SunAnglesData(BaseModel):
|
|
29
|
+
azimuth: SunAngleData
|
|
30
|
+
zenith: SunAngleData
|
|
31
|
+
|
|
32
|
+
def get_angle(self, angle: SunAngle) -> SunAngleData:
|
|
33
|
+
if angle == SunAngle.azimuth:
|
|
34
|
+
return self.azimuth
|
|
35
|
+
elif angle == SunAngle.zenith:
|
|
36
|
+
return self.zenith
|
|
37
|
+
else:
|
|
38
|
+
raise KeyError(f"unknown angle: {angle}")
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class ViewingIncidenceAngle(BaseModel):
|
|
42
|
+
model_config = dict(arbitrary_types_allowed=True)
|
|
43
|
+
detectors: Dict[int, ReferencedRaster]
|
|
44
|
+
mean: float
|
|
45
|
+
|
|
46
|
+
def merge_detectors(
|
|
47
|
+
self, fill_edges: bool = True, smoothing_iterations: int = 3
|
|
48
|
+
) -> ReferencedRaster:
|
|
49
|
+
if not self.detectors:
|
|
50
|
+
raise CorruptedProductMetadata("no viewing incidence angles available")
|
|
51
|
+
sample = next(iter(self.detectors.values()))
|
|
52
|
+
with warnings.catch_warnings():
|
|
53
|
+
warnings.simplefilter("ignore", category=RuntimeWarning)
|
|
54
|
+
merged = np.nanmean(
|
|
55
|
+
np.stack([raster.data for raster in self.detectors.values()]), axis=0
|
|
56
|
+
)
|
|
57
|
+
if fill_edges:
|
|
58
|
+
merged = fillnodata(
|
|
59
|
+
ma.masked_invalid(merged), smoothing_iterations=smoothing_iterations
|
|
60
|
+
)
|
|
61
|
+
return ReferencedRaster.from_array_like(
|
|
62
|
+
array_like=ma.masked_invalid(merged),
|
|
63
|
+
transform=sample.transform,
|
|
64
|
+
crs=sample.crs,
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
class ViewingIncidenceAngles(BaseModel):
|
|
69
|
+
azimuth: ViewingIncidenceAngle
|
|
70
|
+
zenith: ViewingIncidenceAngle
|
|
71
|
+
|
|
72
|
+
def get_angle(self, angle: ViewAngle) -> ViewingIncidenceAngle:
|
|
73
|
+
if angle == ViewAngle.azimuth:
|
|
74
|
+
return self.azimuth
|
|
75
|
+
elif angle == ViewAngle.zenith:
|
|
76
|
+
return self.zenith
|
|
77
|
+
else:
|
|
78
|
+
raise KeyError(f"unknown angle: {angle}")
|
|
@@ -6,14 +6,12 @@ sun angles, quality masks, etc.
|
|
|
6
6
|
from __future__ import annotations
|
|
7
7
|
|
|
8
8
|
import logging
|
|
9
|
-
import warnings
|
|
10
9
|
from functools import cached_property
|
|
11
|
-
from typing import Any,
|
|
10
|
+
from typing import Any, Dict, List, Optional, Tuple, Union
|
|
12
11
|
from xml.etree.ElementTree import Element, ParseError
|
|
13
12
|
|
|
14
13
|
import numpy as np
|
|
15
14
|
import numpy.ma as ma
|
|
16
|
-
from pydantic import BaseModel
|
|
17
15
|
import pystac
|
|
18
16
|
from affine import Affine
|
|
19
17
|
from fiona.transform import transform_geom
|
|
@@ -33,9 +31,17 @@ from tilematrix import Shape
|
|
|
33
31
|
|
|
34
32
|
from mapchete_eo.exceptions import AssetEmpty, AssetMissing, CorruptedProductMetadata
|
|
35
33
|
from mapchete_eo.io import open_xml, read_mask_as_raster
|
|
36
|
-
from mapchete_eo.
|
|
37
|
-
from mapchete_eo.
|
|
38
|
-
from mapchete_eo.platforms.sentinel2.
|
|
34
|
+
from mapchete_eo.io.items import get_item_property
|
|
35
|
+
from mapchete_eo.io.path import asset_mpath
|
|
36
|
+
from mapchete_eo.platforms.sentinel2.metadata_parser.models import (
|
|
37
|
+
ViewingIncidenceAngles,
|
|
38
|
+
SunAngleData,
|
|
39
|
+
SunAnglesData,
|
|
40
|
+
)
|
|
41
|
+
from mapchete_eo.platforms.sentinel2.metadata_parser.base import S2MetadataPathMapper
|
|
42
|
+
from mapchete_eo.platforms.sentinel2.metadata_parser.default_path_mapper import (
|
|
43
|
+
XMLMapper,
|
|
44
|
+
)
|
|
39
45
|
from mapchete_eo.platforms.sentinel2.processing_baseline import ProcessingBaseline
|
|
40
46
|
from mapchete_eo.platforms.sentinel2.types import (
|
|
41
47
|
BandQI,
|
|
@@ -59,77 +65,12 @@ def open_granule_metadata_xml(metadata_xml: MPath) -> Element:
|
|
|
59
65
|
raise CorruptedProductMetadata(exc)
|
|
60
66
|
|
|
61
67
|
|
|
62
|
-
def s2metadata_from_stac_item(
|
|
63
|
-
item: pystac.Item,
|
|
64
|
-
metadata_assets: List[str] = ["metadata", "granule_metadata"],
|
|
65
|
-
boa_offset_fields: List[str] = [
|
|
66
|
-
"sentinel:boa_offset_applied",
|
|
67
|
-
"sentinel2:boa_offset_applied",
|
|
68
|
-
"earthsearch:boa_offset_applied",
|
|
69
|
-
],
|
|
70
|
-
processing_baseline_fields: List[str] = [
|
|
71
|
-
"s2:processing_baseline",
|
|
72
|
-
"sentinel:processing_baseline",
|
|
73
|
-
"sentinel2:processing_baseline",
|
|
74
|
-
"processing:version",
|
|
75
|
-
],
|
|
76
|
-
**kwargs,
|
|
77
|
-
) -> S2Metadata:
|
|
78
|
-
"""Custom code to initialize S2Metadata from a STAC item.
|
|
79
|
-
|
|
80
|
-
Depending on from which catalog the STAC item comes, this function should correctly
|
|
81
|
-
set all custom flags such as BOA offsets or pass on the correct path to the metadata XML
|
|
82
|
-
using the proper asset name.
|
|
83
|
-
"""
|
|
84
|
-
metadata_assets = metadata_assets
|
|
85
|
-
for metadata_asset in metadata_assets:
|
|
86
|
-
if metadata_asset in item.assets:
|
|
87
|
-
metadata_path = MPath(item.assets[metadata_asset].href)
|
|
88
|
-
break
|
|
89
|
-
else: # pragma: no cover
|
|
90
|
-
raise KeyError(
|
|
91
|
-
f"could not find path to metadata XML file in assets: {', '.join(item.assets.keys())}"
|
|
92
|
-
)
|
|
93
|
-
|
|
94
|
-
def _determine_offset():
|
|
95
|
-
for field in boa_offset_fields:
|
|
96
|
-
if item.properties.get(field):
|
|
97
|
-
return True
|
|
98
|
-
|
|
99
|
-
return False
|
|
100
|
-
|
|
101
|
-
boa_offset_applied = _determine_offset()
|
|
102
|
-
|
|
103
|
-
if metadata_path.is_remote() or metadata_path.is_absolute():
|
|
104
|
-
metadata_xml = metadata_path
|
|
105
|
-
else:
|
|
106
|
-
metadata_xml = MPath(item.self_href).parent / metadata_path
|
|
107
|
-
for processing_baseline_field in processing_baseline_fields:
|
|
108
|
-
try:
|
|
109
|
-
processing_baseline = item.properties[processing_baseline_field]
|
|
110
|
-
break
|
|
111
|
-
except KeyError:
|
|
112
|
-
pass
|
|
113
|
-
else: # pragma: no cover
|
|
114
|
-
raise KeyError(
|
|
115
|
-
f"could not find processing baseline version in item properties: {item.properties}"
|
|
116
|
-
)
|
|
117
|
-
return S2Metadata.from_metadata_xml(
|
|
118
|
-
metadata_xml=metadata_xml,
|
|
119
|
-
processing_baseline=processing_baseline,
|
|
120
|
-
boa_offset_applied=boa_offset_applied,
|
|
121
|
-
**kwargs,
|
|
122
|
-
)
|
|
123
|
-
|
|
124
|
-
|
|
125
68
|
class S2Metadata:
|
|
126
69
|
metadata_xml: MPath
|
|
127
|
-
path_mapper:
|
|
70
|
+
path_mapper: S2MetadataPathMapper
|
|
128
71
|
processing_baseline: ProcessingBaseline
|
|
129
72
|
boa_offset_applied: bool = False
|
|
130
73
|
_cached_xml_root: Optional[Element] = None
|
|
131
|
-
path_mapper_guesser: Callable = default_path_mapper_guesser
|
|
132
|
-
from_stac_item_constructor: Callable = s2metadata_from_stac_item
|
|
133
74
|
crs: CRS
|
|
134
75
|
bounds: Bounds
|
|
135
76
|
footprint: Union[Polygon, MultiPolygon]
|
|
@@ -138,7 +79,7 @@ class S2Metadata:
|
|
|
138
79
|
def __init__(
|
|
139
80
|
self,
|
|
140
81
|
metadata_xml: MPath,
|
|
141
|
-
path_mapper:
|
|
82
|
+
path_mapper: S2MetadataPathMapper,
|
|
142
83
|
xml_root: Optional[Element] = None,
|
|
143
84
|
boa_offset_applied: bool = False,
|
|
144
85
|
**kwargs,
|
|
@@ -186,19 +127,15 @@ class S2Metadata:
|
|
|
186
127
|
def from_metadata_xml(
|
|
187
128
|
cls,
|
|
188
129
|
metadata_xml: Union[str, MPath],
|
|
130
|
+
path_mapper: Optional[S2MetadataPathMapper] = None,
|
|
189
131
|
processing_baseline: Optional[str] = None,
|
|
190
|
-
path_mapper: Optional[S2PathMapper] = None,
|
|
191
132
|
**kwargs,
|
|
192
133
|
) -> S2Metadata:
|
|
193
134
|
metadata_xml = MPath.from_inp(metadata_xml, **kwargs)
|
|
194
135
|
xml_root = open_granule_metadata_xml(metadata_xml)
|
|
136
|
+
|
|
195
137
|
if path_mapper is None:
|
|
196
|
-
|
|
197
|
-
path_mapper = cls.path_mapper_guesser(
|
|
198
|
-
metadata_xml,
|
|
199
|
-
xml_root=xml_root,
|
|
200
|
-
**kwargs,
|
|
201
|
-
)
|
|
138
|
+
path_mapper = XMLMapper(metadata_xml=metadata_xml, xml_root=xml_root)
|
|
202
139
|
|
|
203
140
|
# use processing baseline version from argument if available
|
|
204
141
|
if processing_baseline:
|
|
@@ -217,9 +154,38 @@ class S2Metadata:
|
|
|
217
154
|
metadata_xml, path_mapper=path_mapper, xml_root=xml_root, **kwargs
|
|
218
155
|
)
|
|
219
156
|
|
|
220
|
-
@
|
|
221
|
-
def from_stac_item(
|
|
222
|
-
|
|
157
|
+
@staticmethod
|
|
158
|
+
def from_stac_item(
|
|
159
|
+
item: pystac.Item,
|
|
160
|
+
metadata_xml_asset_name: Tuple[str, ...] = ("metadata", "granule_metadata"),
|
|
161
|
+
boa_offset_field: Union[str, Tuple[str, ...]] = (
|
|
162
|
+
"earthsearch:boa_offset_applied"
|
|
163
|
+
),
|
|
164
|
+
processing_baseline_field: Union[str, Tuple[str, ...]] = (
|
|
165
|
+
"s2:processing_baseline",
|
|
166
|
+
"sentinel2:processing_baseline",
|
|
167
|
+
"processing:version",
|
|
168
|
+
),
|
|
169
|
+
**kwargs,
|
|
170
|
+
) -> S2Metadata:
|
|
171
|
+
# try to find path to metadata.xml
|
|
172
|
+
metadata_xml_path = asset_mpath(item, metadata_xml_asset_name)
|
|
173
|
+
# make path absolute
|
|
174
|
+
if not (metadata_xml_path.is_remote() or metadata_xml_path.is_absolute()):
|
|
175
|
+
metadata_xml_path = MPath(item.self_href).parent / metadata_xml_path
|
|
176
|
+
|
|
177
|
+
# try to find information on processing baseline version
|
|
178
|
+
processing_baseline = get_item_property(item, processing_baseline_field)
|
|
179
|
+
|
|
180
|
+
# see if boa_offset_applied flag is available
|
|
181
|
+
boa_offset_applied = get_item_property(item, boa_offset_field, default=False)
|
|
182
|
+
|
|
183
|
+
return S2Metadata.from_metadata_xml(
|
|
184
|
+
metadata_xml=metadata_xml_path,
|
|
185
|
+
processing_baseline=processing_baseline,
|
|
186
|
+
boa_offset_applied=boa_offset_applied,
|
|
187
|
+
**kwargs,
|
|
188
|
+
)
|
|
223
189
|
|
|
224
190
|
@property
|
|
225
191
|
def xml_root(self):
|
|
@@ -270,13 +236,13 @@ class S2Metadata:
|
|
|
270
236
|
for product_qi_mask in ProductQI:
|
|
271
237
|
if product_qi_mask == ProductQI.classification:
|
|
272
238
|
out[product_qi_mask.name] = self.path_mapper.product_qi_mask(
|
|
273
|
-
product_qi_mask
|
|
239
|
+
qi_mask=product_qi_mask
|
|
274
240
|
)
|
|
275
241
|
else:
|
|
276
242
|
for resolution in ProductQIMaskResolution:
|
|
277
243
|
out[f"{product_qi_mask.name}-{resolution.name}"] = (
|
|
278
244
|
self.path_mapper.product_qi_mask(
|
|
279
|
-
product_qi_mask, resolution=resolution
|
|
245
|
+
qi_mask=product_qi_mask, resolution=resolution
|
|
280
246
|
)
|
|
281
247
|
)
|
|
282
248
|
|
|
@@ -587,65 +553,6 @@ class S2Metadata:
|
|
|
587
553
|
return mean
|
|
588
554
|
|
|
589
555
|
|
|
590
|
-
class SunAngleData(BaseModel):
|
|
591
|
-
model_config = dict(arbitrary_types_allowed=True)
|
|
592
|
-
raster: ReferencedRaster
|
|
593
|
-
mean: float
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
class SunAnglesData(BaseModel):
|
|
597
|
-
azimuth: SunAngleData
|
|
598
|
-
zenith: SunAngleData
|
|
599
|
-
|
|
600
|
-
def get_angle(self, angle: SunAngle) -> SunAngleData:
|
|
601
|
-
if angle == SunAngle.azimuth:
|
|
602
|
-
return self.azimuth
|
|
603
|
-
elif angle == SunAngle.zenith:
|
|
604
|
-
return self.zenith
|
|
605
|
-
else:
|
|
606
|
-
raise KeyError(f"unknown angle: {angle}")
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
class ViewingIncidenceAngle(BaseModel):
|
|
610
|
-
model_config = dict(arbitrary_types_allowed=True)
|
|
611
|
-
detectors: Dict[int, ReferencedRaster]
|
|
612
|
-
mean: float
|
|
613
|
-
|
|
614
|
-
def merge_detectors(
|
|
615
|
-
self, fill_edges: bool = True, smoothing_iterations: int = 3
|
|
616
|
-
) -> ReferencedRaster:
|
|
617
|
-
if not self.detectors:
|
|
618
|
-
raise CorruptedProductMetadata("no viewing incidence angles available")
|
|
619
|
-
sample = next(iter(self.detectors.values()))
|
|
620
|
-
with warnings.catch_warnings():
|
|
621
|
-
warnings.simplefilter("ignore", category=RuntimeWarning)
|
|
622
|
-
merged = np.nanmean(
|
|
623
|
-
np.stack([raster.data for raster in self.detectors.values()]), axis=0
|
|
624
|
-
)
|
|
625
|
-
if fill_edges:
|
|
626
|
-
merged = fillnodata(
|
|
627
|
-
ma.masked_invalid(merged), smoothing_iterations=smoothing_iterations
|
|
628
|
-
)
|
|
629
|
-
return ReferencedRaster.from_array_like(
|
|
630
|
-
array_like=ma.masked_invalid(merged),
|
|
631
|
-
transform=sample.transform,
|
|
632
|
-
crs=sample.crs,
|
|
633
|
-
)
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
class ViewingIncidenceAngles(BaseModel):
|
|
637
|
-
azimuth: ViewingIncidenceAngle
|
|
638
|
-
zenith: ViewingIncidenceAngle
|
|
639
|
-
|
|
640
|
-
def get_angle(self, angle: ViewAngle) -> ViewingIncidenceAngle:
|
|
641
|
-
if angle == ViewAngle.azimuth:
|
|
642
|
-
return self.azimuth
|
|
643
|
-
elif angle == ViewAngle.zenith:
|
|
644
|
-
return self.zenith
|
|
645
|
-
else:
|
|
646
|
-
raise KeyError(f"unknown angle: {angle}")
|
|
647
|
-
|
|
648
|
-
|
|
649
556
|
def _get_grids(root: Element, crs: CRS) -> Dict[Resolution, Grid]:
|
|
650
557
|
geoinfo = {
|
|
651
558
|
Resolution["10m"]: dict(crs=crs),
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
from typing import Dict, Any
|
|
2
|
+
|
|
3
|
+
# importing this is crucial so the mapping functions get registered before registry is accessed
|
|
4
|
+
from mapchete_eo.platforms.sentinel2.preconfigured_sources.item_mappers import (
|
|
5
|
+
earthsearch_assets_paths_mapper,
|
|
6
|
+
earthsearch_id_mapper,
|
|
7
|
+
earthsearch_to_s2metadata,
|
|
8
|
+
cdse_asset_names,
|
|
9
|
+
cdse_s2metadata,
|
|
10
|
+
)
|
|
11
|
+
from mapchete_eo.platforms.sentinel2.preconfigured_sources.guessers import (
|
|
12
|
+
guess_metadata_path_mapper,
|
|
13
|
+
guess_s2metadata_from_item,
|
|
14
|
+
guess_s2metadata_from_metadata_xml,
|
|
15
|
+
)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
__all__ = [
|
|
19
|
+
"guess_metadata_path_mapper",
|
|
20
|
+
"guess_s2metadata_from_item",
|
|
21
|
+
"guess_s2metadata_from_metadata_xml",
|
|
22
|
+
"earthsearch_assets_paths_mapper",
|
|
23
|
+
"earthsearch_id_mapper",
|
|
24
|
+
"earthsearch_to_s2metadata",
|
|
25
|
+
"cdse_asset_names",
|
|
26
|
+
"cdse_s2metadata",
|
|
27
|
+
]
|
|
28
|
+
|
|
29
|
+
KNOWN_SOURCES: Dict[str, Any] = {
|
|
30
|
+
"EarthSearch": {
|
|
31
|
+
"collection": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-c1-l2a",
|
|
32
|
+
},
|
|
33
|
+
"EarthSearch_legacy": {
|
|
34
|
+
"collection": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a",
|
|
35
|
+
},
|
|
36
|
+
"CDSE": {
|
|
37
|
+
"collection": "https://stac.dataspace.copernicus.eu/v1/collections/sentinel-2-l2a",
|
|
38
|
+
"metadata_archive": "CDSE",
|
|
39
|
+
},
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
DEPRECATED_ARCHIVES = {
|
|
43
|
+
"S2AWS_COG": {
|
|
44
|
+
"collection": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-c1-l2a",
|
|
45
|
+
},
|
|
46
|
+
"S2AWS_JP2": {
|
|
47
|
+
"collection": "https://stac.dataspace.copernicus.eu/v1/collections/sentinel-2-l2a",
|
|
48
|
+
"data_archive": "AWSJP2",
|
|
49
|
+
},
|
|
50
|
+
"S2CDSE_AWSJP2": {
|
|
51
|
+
"collection": "https://stac.dataspace.copernicus.eu/v1/collections/sentinel-2-l2a",
|
|
52
|
+
"data_archive": "AWSJP2",
|
|
53
|
+
},
|
|
54
|
+
"S2CDSE_JP2": {
|
|
55
|
+
"collection": "https://stac.dataspace.copernicus.eu/v1/collections/sentinel-2-l2a",
|
|
56
|
+
},
|
|
57
|
+
}
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
from typing import List
|
|
2
|
+
|
|
3
|
+
from mapchete.path import MPathLike, MPath
|
|
4
|
+
from pystac import Item
|
|
5
|
+
|
|
6
|
+
from mapchete_eo.platforms.sentinel2.metadata_parser.base import S2MetadataPathMapper
|
|
7
|
+
from mapchete_eo.platforms.sentinel2.metadata_parser.default_path_mapper import (
|
|
8
|
+
XMLMapper,
|
|
9
|
+
)
|
|
10
|
+
from mapchete_eo.platforms.sentinel2.metadata_parser.s2metadata import S2Metadata
|
|
11
|
+
from mapchete_eo.platforms.sentinel2.preconfigured_sources.metadata_xml_mappers import (
|
|
12
|
+
EarthSearchPathMapper,
|
|
13
|
+
SinergisePathMapper,
|
|
14
|
+
)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def guess_metadata_path_mapper(
|
|
18
|
+
metadata_xml: MPathLike, **kwargs
|
|
19
|
+
) -> S2MetadataPathMapper:
|
|
20
|
+
"""Guess S2PathMapper based on URL.
|
|
21
|
+
|
|
22
|
+
If a new path mapper is added in this module, it should also be added to this function
|
|
23
|
+
in order to be detected.
|
|
24
|
+
"""
|
|
25
|
+
metadata_xml = MPath.from_inp(metadata_xml)
|
|
26
|
+
if metadata_xml.startswith(
|
|
27
|
+
("https://roda.sentinel-hub.com/sentinel-s2-l2a/", "s3://sentinel-s2-l2a/")
|
|
28
|
+
) or metadata_xml.startswith(
|
|
29
|
+
("https://roda.sentinel-hub.com/sentinel-s2-l1c/", "s3://sentinel-s2-l1c/")
|
|
30
|
+
):
|
|
31
|
+
return SinergisePathMapper(metadata_xml, **kwargs)
|
|
32
|
+
elif metadata_xml.startswith(
|
|
33
|
+
"https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/"
|
|
34
|
+
):
|
|
35
|
+
return EarthSearchPathMapper(metadata_xml, **kwargs)
|
|
36
|
+
else:
|
|
37
|
+
return XMLMapper(metadata_xml, **kwargs)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def guess_s2metadata_from_metadata_xml(metadata_xml: MPathLike, **kwargs) -> S2Metadata:
|
|
41
|
+
return S2Metadata.from_metadata_xml(
|
|
42
|
+
metadata_xml=metadata_xml,
|
|
43
|
+
path_mapper=guess_metadata_path_mapper(metadata_xml, **kwargs),
|
|
44
|
+
**kwargs,
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def guess_s2metadata_from_item(
|
|
49
|
+
item: Item,
|
|
50
|
+
metadata_assets: List[str] = ["metadata", "granule_metadata"],
|
|
51
|
+
boa_offset_fields: List[str] = [
|
|
52
|
+
"sentinel:boa_offset_applied",
|
|
53
|
+
"sentinel2:boa_offset_applied",
|
|
54
|
+
"earthsearch:boa_offset_applied",
|
|
55
|
+
],
|
|
56
|
+
processing_baseline_fields: List[str] = [
|
|
57
|
+
"s2:processing_baseline",
|
|
58
|
+
"sentinel:processing_baseline",
|
|
59
|
+
"sentinel2:processing_baseline",
|
|
60
|
+
"processing:version",
|
|
61
|
+
],
|
|
62
|
+
**kwargs,
|
|
63
|
+
) -> S2Metadata:
|
|
64
|
+
"""Custom code to initialize S2Metadata from a STAC item.
|
|
65
|
+
|
|
66
|
+
Depending on from which catalog the STAC item comes, this function should correctly
|
|
67
|
+
set all custom flags such as BOA offsets or pass on the correct path to the metadata XML
|
|
68
|
+
using the proper asset name.
|
|
69
|
+
"""
|
|
70
|
+
metadata_assets = metadata_assets
|
|
71
|
+
for metadata_asset in metadata_assets:
|
|
72
|
+
if metadata_asset in item.assets:
|
|
73
|
+
metadata_path = MPath(item.assets[metadata_asset].href)
|
|
74
|
+
break
|
|
75
|
+
else: # pragma: no cover
|
|
76
|
+
raise KeyError(
|
|
77
|
+
f"could not find path to metadata XML file in assets: {', '.join(item.assets.keys())}"
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
def _determine_offset():
|
|
81
|
+
for field in boa_offset_fields:
|
|
82
|
+
if item.properties.get(field):
|
|
83
|
+
return True
|
|
84
|
+
|
|
85
|
+
return False
|
|
86
|
+
|
|
87
|
+
boa_offset_applied = _determine_offset()
|
|
88
|
+
|
|
89
|
+
if metadata_path.is_remote() or metadata_path.is_absolute():
|
|
90
|
+
metadata_xml = metadata_path
|
|
91
|
+
else:
|
|
92
|
+
metadata_xml = MPath(item.self_href).parent / metadata_path
|
|
93
|
+
for processing_baseline_field in processing_baseline_fields:
|
|
94
|
+
try:
|
|
95
|
+
processing_baseline = item.properties[processing_baseline_field]
|
|
96
|
+
break
|
|
97
|
+
except KeyError:
|
|
98
|
+
pass
|
|
99
|
+
else: # pragma: no cover
|
|
100
|
+
raise KeyError(
|
|
101
|
+
f"could not find processing baseline version in item properties: {item.properties}"
|
|
102
|
+
)
|
|
103
|
+
return guess_s2metadata_from_metadata_xml(
|
|
104
|
+
metadata_xml,
|
|
105
|
+
processing_baseline=processing_baseline,
|
|
106
|
+
boa_offset_applied=boa_offset_applied,
|
|
107
|
+
**kwargs,
|
|
108
|
+
)
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
from mapchete.path import MPath
|
|
2
|
+
from pystac import Item
|
|
3
|
+
|
|
4
|
+
from mapchete_eo.platforms.sentinel2._mapper_registry import (
|
|
5
|
+
maps_item_id,
|
|
6
|
+
maps_stac_metadata,
|
|
7
|
+
creates_s2metadata,
|
|
8
|
+
)
|
|
9
|
+
from mapchete_eo.platforms.sentinel2.preconfigured_sources.metadata_xml_mappers import (
|
|
10
|
+
CDSEPathMapper,
|
|
11
|
+
EarthSearchPathMapper,
|
|
12
|
+
EarthSearchC1PathMapper,
|
|
13
|
+
SinergisePathMapper,
|
|
14
|
+
)
|
|
15
|
+
from mapchete_eo.platforms.sentinel2.metadata_parser.s2metadata import S2Metadata
|
|
16
|
+
from mapchete_eo.search.s2_mgrs import S2Tile
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
# mapper functions decorated with metadata to have driver decide which one to apply when #
|
|
20
|
+
##########################################################################################
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@maps_item_id(from_collections=["EarthSearch", "EarthSearch_legacy"])
|
|
24
|
+
def earthsearch_id_mapper(item: Item) -> Item:
|
|
25
|
+
item.id = item.properties["s2:product_uri"].rstrip(".SAFE")
|
|
26
|
+
return item
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@maps_stac_metadata(from_collections=["EarthSearch"], to_data_archives=["AWSCOG"])
|
|
30
|
+
def earthsearch_assets_paths_mapper(item: Item) -> Item:
|
|
31
|
+
"""Nothing to do here as paths match catalog."""
|
|
32
|
+
return item
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@creates_s2metadata(from_collections=["EarthSearch"], to_metadata_archives=["roda"])
|
|
36
|
+
def earthsearch_to_s2metadata(item: Item) -> S2Metadata:
|
|
37
|
+
return S2Metadata.from_stac_item(
|
|
38
|
+
item,
|
|
39
|
+
path_mapper=EarthSearchC1PathMapper(
|
|
40
|
+
MPath(item.assets["granule_metadata"].href)
|
|
41
|
+
),
|
|
42
|
+
processing_baseline_field="s2:processing_baseline",
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
@creates_s2metadata(
|
|
47
|
+
from_collections=["EarthSearch_legacy"], to_metadata_archives=["roda"]
|
|
48
|
+
)
|
|
49
|
+
def earthsearch_legacy_to_s2metadata(item: Item) -> S2Metadata:
|
|
50
|
+
return S2Metadata.from_stac_item(
|
|
51
|
+
item,
|
|
52
|
+
path_mapper=EarthSearchPathMapper(MPath(item.assets["granule_metadata"].href)),
|
|
53
|
+
boa_offset_field="earthsearch:boa_offset_applied",
|
|
54
|
+
processing_baseline_field="s2:processing_baseline",
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
@maps_item_id(from_collections=["CDSE"])
|
|
59
|
+
def plain_id_mapper(item: Item) -> Item:
|
|
60
|
+
return item
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
CDSE_ASSET_NAME_MAPPING = {
|
|
64
|
+
"AOT_10m": "aot",
|
|
65
|
+
"B01_20m": "coastal",
|
|
66
|
+
"B02_10m": "blue",
|
|
67
|
+
"B03_10m": "green",
|
|
68
|
+
"B04_10m": "red",
|
|
69
|
+
"B05_20m": "rededge1",
|
|
70
|
+
"B06_20m": "rededge2",
|
|
71
|
+
"B07_20m": "rededge3",
|
|
72
|
+
"B08_10m": "nir",
|
|
73
|
+
"B09_60m": "nir09",
|
|
74
|
+
"B11_20m": "swir16",
|
|
75
|
+
"B12_20m": "swir22",
|
|
76
|
+
"B8A_20m": "nir08",
|
|
77
|
+
"SCL_20m": "scl",
|
|
78
|
+
"TCI_10m": "visual",
|
|
79
|
+
"WVP_10m": "wvp",
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
@maps_stac_metadata(from_collections=["CDSE"])
|
|
84
|
+
def cdse_asset_names(item: Item) -> Item:
|
|
85
|
+
new_assets = {}
|
|
86
|
+
for asset_name, asset in item.assets.items():
|
|
87
|
+
if asset_name in CDSE_ASSET_NAME_MAPPING:
|
|
88
|
+
asset_name = CDSE_ASSET_NAME_MAPPING[asset_name]
|
|
89
|
+
new_assets[asset_name] = asset
|
|
90
|
+
|
|
91
|
+
item.assets = new_assets
|
|
92
|
+
|
|
93
|
+
item.properties["s2:datastrip_id"] = item.properties.get("eopf:datastrip_id")
|
|
94
|
+
return item
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
@maps_stac_metadata(from_collections=["CDSE"], to_data_archives=["AWSJP2"])
|
|
98
|
+
def map_cdse_paths_to_jp2_archive(item: Item) -> Item:
|
|
99
|
+
"""
|
|
100
|
+
CSDE has the following assets:
|
|
101
|
+
AOT_10m, AOT_20m, AOT_60m, B01_20m, B01_60m, B02_10m, B02_20m, B02_60m, B03_10m, B03_20m,
|
|
102
|
+
B03_60m, B04_10m, B04_20m, B04_60m, B05_20m, B05_60m, B06_20m, B06_60m, B07_20m, B07_60m,
|
|
103
|
+
B08_10m, B09_60m, B11_20m, B11_60m, B12_20m, B12_60m, B8A_20m, B8A_60m, Product, SCL_20m,
|
|
104
|
+
SCL_60m, TCI_10m, TCI_20m, TCI_60m, WVP_10m, WVP_20m, WVP_60m, thumbnail, safe_manifest,
|
|
105
|
+
granule_metadata, inspire_metadata, product_metadata, datastrip_metadata
|
|
106
|
+
|
|
107
|
+
sample path for AWS JP2:
|
|
108
|
+
s3://sentinel-s2-l2a/tiles/51/K/XR/2020/7/31/0/R10m/
|
|
109
|
+
"""
|
|
110
|
+
if item.datetime is None:
|
|
111
|
+
raise ValueError(f"product {item.get_self_href()} does not have a timestamp")
|
|
112
|
+
path_base_scheme = "s3://sentinel-s2-l2a/tiles/{utm_zone}/{latitude_band}/{grid_square}/{year}/{month}/{day}/{count}"
|
|
113
|
+
s2tile = S2Tile.from_grid_code(item.properties["grid:code"])
|
|
114
|
+
product_basepath = MPath(
|
|
115
|
+
path_base_scheme.format(
|
|
116
|
+
utm_zone=int(s2tile.utm_zone),
|
|
117
|
+
latitude_band=s2tile.latitude_band,
|
|
118
|
+
grid_square=s2tile.grid_square,
|
|
119
|
+
year=item.datetime.year,
|
|
120
|
+
month=item.datetime.month,
|
|
121
|
+
day=item.datetime.day,
|
|
122
|
+
count=0, # TODO: get count dynamically from metadata
|
|
123
|
+
)
|
|
124
|
+
)
|
|
125
|
+
new_assets = {}
|
|
126
|
+
for asset_name, asset in item.assets.items():
|
|
127
|
+
# ignore these assets
|
|
128
|
+
if asset_name in [
|
|
129
|
+
"Product",
|
|
130
|
+
"safe_manifest",
|
|
131
|
+
"product_metadata",
|
|
132
|
+
"inspire_metadata",
|
|
133
|
+
"datastrip_metadata",
|
|
134
|
+
]:
|
|
135
|
+
continue
|
|
136
|
+
# set thumbnnail
|
|
137
|
+
elif asset_name == "thumbnail":
|
|
138
|
+
asset.href = str(product_basepath / "R60m" / "TCI.jp2")
|
|
139
|
+
# point to proper metadata
|
|
140
|
+
elif asset_name == "granule_metadata":
|
|
141
|
+
asset.href = str(product_basepath / "metadata.xml")
|
|
142
|
+
# change band asset names and point to their new locations
|
|
143
|
+
elif asset_name in CDSE_ASSET_NAME_MAPPING:
|
|
144
|
+
name, resolution = asset_name.split("_")
|
|
145
|
+
asset.href = product_basepath / f"R{resolution}" / f"{name}.jp2"
|
|
146
|
+
asset_name = CDSE_ASSET_NAME_MAPPING[asset_name]
|
|
147
|
+
else:
|
|
148
|
+
continue
|
|
149
|
+
new_assets[asset_name] = asset
|
|
150
|
+
|
|
151
|
+
item.assets = new_assets
|
|
152
|
+
|
|
153
|
+
return item
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
@creates_s2metadata(from_collections=["CDSE"], to_metadata_archives=["CDSE"])
|
|
157
|
+
def cdse_s2metadata(item: Item) -> S2Metadata:
|
|
158
|
+
return S2Metadata.from_stac_item(
|
|
159
|
+
item,
|
|
160
|
+
path_mapper=CDSEPathMapper(MPath(item.assets["granule_metadata"].href)),
|
|
161
|
+
processing_baseline_field="processing:version",
|
|
162
|
+
)
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
@creates_s2metadata(from_collections=["CDSE"], to_metadata_archives=["roda"])
|
|
166
|
+
def cdse_to_roda_s2metadata(item: Item) -> S2Metadata:
|
|
167
|
+
return S2Metadata.from_stac_item(
|
|
168
|
+
item,
|
|
169
|
+
path_mapper=SinergisePathMapper(MPath(item.assets["granule_metadata"].href)),
|
|
170
|
+
processing_baseline_field="processing:version",
|
|
171
|
+
)
|