mapchete-eo 2026.2.0__tar.gz → 2026.11.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/PKG-INFO +5 -3
  2. mapchete_eo-2026.11.0/mapchete_eo/__init__.py +1 -0
  3. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/base.py +22 -6
  4. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/io/assets.py +5 -5
  5. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/io/levelled_cubes.py +9 -6
  6. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/io/path.py +1 -21
  7. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/io/products.py +20 -8
  8. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/platforms/sentinel2/masks.py +6 -3
  9. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/platforms/sentinel2/preconfigured_sources/item_mappers.py +2 -2
  10. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/platforms/sentinel2/product.py +4 -2
  11. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/search/stac_search.py +1 -1
  12. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/pyproject.toml +18 -5
  13. mapchete_eo-2026.2.0/mapchete_eo/__init__.py +0 -1
  14. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/.gitignore +0 -0
  15. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/LICENSE +0 -0
  16. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/README.rst +0 -0
  17. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/array/__init__.py +0 -0
  18. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/array/buffer.py +0 -0
  19. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/array/color.py +0 -0
  20. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/array/convert.py +0 -0
  21. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/blacklist.txt +0 -0
  22. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/cli/__init__.py +0 -0
  23. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/cli/bounds.py +0 -0
  24. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/cli/options_arguments.py +0 -0
  25. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/cli/s2_brdf.py +0 -0
  26. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/cli/s2_cat_results.py +0 -0
  27. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/cli/s2_find_broken_products.py +0 -0
  28. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/cli/s2_jp2_static_catalog.py +0 -0
  29. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/cli/s2_mask.py +0 -0
  30. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/cli/s2_mgrs.py +0 -0
  31. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/cli/s2_rgb.py +0 -0
  32. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/cli/s2_verify.py +0 -0
  33. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/cli/static_catalog.py +0 -0
  34. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/eostac.py +0 -0
  35. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/exceptions.py +0 -0
  36. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/image_operations/__init__.py +0 -0
  37. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/image_operations/blend_functions.py +0 -0
  38. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/image_operations/color_correction.py +0 -0
  39. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/image_operations/compositing.py +0 -0
  40. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/image_operations/dtype_scale.py +0 -0
  41. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/image_operations/fillnodata.py +0 -0
  42. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/image_operations/filters.py +0 -0
  43. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/image_operations/linear_normalization.py +0 -0
  44. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/image_operations/sigmoidal.py +0 -0
  45. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/io/__init__.py +0 -0
  46. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/io/items.py +0 -0
  47. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/io/profiles.py +0 -0
  48. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/platforms/sentinel2/__init__.py +0 -0
  49. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/platforms/sentinel2/_mapper_registry.py +0 -0
  50. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/platforms/sentinel2/bandpass_adjustment.py +0 -0
  51. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/platforms/sentinel2/brdf/__init__.py +0 -0
  52. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/platforms/sentinel2/brdf/config.py +0 -0
  53. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/platforms/sentinel2/brdf/correction.py +0 -0
  54. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/platforms/sentinel2/brdf/hls.py +0 -0
  55. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/platforms/sentinel2/brdf/models.py +0 -0
  56. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/platforms/sentinel2/brdf/protocols.py +0 -0
  57. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/platforms/sentinel2/brdf/ross_thick.py +0 -0
  58. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/platforms/sentinel2/brdf/sun_angle_arrays.py +0 -0
  59. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/platforms/sentinel2/config.py +0 -0
  60. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/platforms/sentinel2/driver.py +0 -0
  61. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/platforms/sentinel2/metadata_parser/__init__.py +0 -0
  62. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/platforms/sentinel2/metadata_parser/base.py +0 -0
  63. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/platforms/sentinel2/metadata_parser/default_path_mapper.py +0 -0
  64. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/platforms/sentinel2/metadata_parser/models.py +0 -0
  65. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/platforms/sentinel2/metadata_parser/s2metadata.py +0 -0
  66. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/platforms/sentinel2/preconfigured_sources/__init__.py +0 -0
  67. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/platforms/sentinel2/preconfigured_sources/guessers.py +0 -0
  68. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/platforms/sentinel2/preconfigured_sources/metadata_xml_mappers.py +0 -0
  69. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/platforms/sentinel2/preprocessing_tasks.py +0 -0
  70. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/platforms/sentinel2/processing_baseline.py +0 -0
  71. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/platforms/sentinel2/source.py +0 -0
  72. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/platforms/sentinel2/types.py +0 -0
  73. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/processes/__init__.py +0 -0
  74. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/processes/config.py +0 -0
  75. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/processes/dtype_scale.py +0 -0
  76. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/processes/eo_to_xarray.py +0 -0
  77. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/processes/merge_rasters.py +0 -0
  78. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/product.py +0 -0
  79. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/protocols.py +0 -0
  80. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/search/__init__.py +0 -0
  81. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/search/base.py +0 -0
  82. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/search/config.py +0 -0
  83. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/search/s2_mgrs.py +0 -0
  84. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/search/stac_static.py +0 -0
  85. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/search/utm_search.py +0 -0
  86. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/settings.py +0 -0
  87. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/sort.py +0 -0
  88. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/source.py +0 -0
  89. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/time.py +0 -0
  90. {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/types.py +0 -0
@@ -1,6 +1,6 @@
1
- Metadata-Version: 2.4
1
+ Metadata-Version: 2.5
2
2
  Name: mapchete-eo
3
- Version: 2026.2.0
3
+ Version: 2026.11.0
4
4
  Summary: mapchete EO data reader
5
5
  Project-URL: Homepage, https://gitlab.eox.at/maps/mapchete_eo
6
6
  Author-email: Joachim Ungar <joachim.ungar@eox.at>, Petr Sevcik <petr.sevcik@eox.at>
@@ -35,9 +35,11 @@ Requires-Dist: myst-parser; extra == 'docs'
35
35
  Requires-Dist: sphinx-rtd-theme>=3.0.2; extra == 'docs'
36
36
  Requires-Dist: sphinx>=8.0; extra == 'docs'
37
37
  Provides-Extra: test
38
+ Requires-Dist: httpx; extra == 'test'
38
39
  Requires-Dist: pytest; extra == 'test'
39
- Requires-Dist: pytest-coverage; extra == 'test'
40
+ Requires-Dist: pytest-cov; extra == 'test'
40
41
  Requires-Dist: pytest-lazy-fixtures; extra == 'test'
42
+ Requires-Dist: pytest-split; extra == 'test'
41
43
  Description-Content-Type: text/x-rst
42
44
 
43
45
  .. image:: logo/mapchete_eo.svg
@@ -0,0 +1 @@
1
+ __version__ = "2026.11.0"
@@ -162,6 +162,7 @@ class EODataCube(base.InputTile):
162
162
  sort: Optional[SortMethodConfig] = None,
163
163
  nodatavals: NodataVals = None,
164
164
  raise_empty: bool = True,
165
+ clear_caches: bool = True,
165
166
  **kwargs,
166
167
  ) -> xr.Dataset:
167
168
  """
@@ -180,6 +181,7 @@ class EODataCube(base.InputTile):
180
181
  raise_empty=raise_empty,
181
182
  product_read_kwargs=kwargs,
182
183
  sort=sort,
184
+ clear_caches=clear_caches,
183
185
  **self.default_read_values(
184
186
  merge_products_by=merge_products_by,
185
187
  merge_method=merge_method,
@@ -202,6 +204,7 @@ class EODataCube(base.InputTile):
202
204
  sort: Optional[SortMethodConfig] = None,
203
205
  nodatavals: NodataVals = None,
204
206
  raise_empty: bool = True,
207
+ clear_caches: bool = True,
205
208
  **kwargs,
206
209
  ) -> ma.MaskedArray:
207
210
  """
@@ -220,6 +223,7 @@ class EODataCube(base.InputTile):
220
223
  product_read_kwargs=kwargs,
221
224
  raise_empty=raise_empty,
222
225
  sort=sort,
226
+ clear_caches=clear_caches,
223
227
  **self.default_read_values(
224
228
  merge_products_by=merge_products_by,
225
229
  merge_method=merge_method,
@@ -247,6 +251,7 @@ class EODataCube(base.InputTile):
247
251
  band_axis_name: str = "bands",
248
252
  x_axis_name: str = "x",
249
253
  y_axis_name: str = "y",
254
+ clear_caches: bool = True,
250
255
  **kwargs,
251
256
  ) -> xr.Dataset:
252
257
  return read_levelled_cube_to_xarray(
@@ -267,6 +272,7 @@ class EODataCube(base.InputTile):
267
272
  x_axis_name=x_axis_name,
268
273
  y_axis_name=y_axis_name,
269
274
  sort=sort,
275
+ clear_caches=clear_caches,
270
276
  **self.default_read_values(
271
277
  merge_products_by=merge_products_by,
272
278
  merge_method=merge_method,
@@ -290,6 +296,7 @@ class EODataCube(base.InputTile):
290
296
  merge_method: Optional[MergeMethod] = None,
291
297
  sort: SortMethodConfig = TargetDateSort(),
292
298
  raise_empty: bool = True,
299
+ clear_caches: bool = True,
293
300
  **kwargs,
294
301
  ) -> ma.MaskedArray:
295
302
  """
@@ -327,6 +334,7 @@ class EODataCube(base.InputTile):
327
334
  raise_empty=raise_empty,
328
335
  product_read_kwargs=kwargs,
329
336
  sort=sort,
337
+ clear_caches=clear_caches,
330
338
  **self.default_read_values(
331
339
  merge_products_by=merge_products_by,
332
340
  merge_method=merge_method,
@@ -500,12 +508,6 @@ class InputData(base.InputData):
500
508
  self.area = self._init_area(input_params)
501
509
  self.time = self.params.time
502
510
 
503
- self.eo_bands = [
504
- eo_band
505
- for source in self.params.source
506
- for eo_band in source.eo_bands(base_dir=self.conf_dir)
507
- ]
508
-
509
511
  if self.readonly: # pragma: no cover
510
512
  return
511
513
  # don't use preprocessing tasks for Sentinel-2 products:
@@ -533,6 +535,17 @@ class InputData(base.InputData):
533
535
  ]
534
536
  )
535
537
 
538
+ @cached_property
539
+ def eo_bands(self) -> List[str]:
540
+ if self.area.is_empty:
541
+ logger.debug("Input area is empty, nothing to be found here.")
542
+ return []
543
+ return [
544
+ eo_band
545
+ for source in self.params.source
546
+ for eo_band in source.eo_bands(base_dir=self.conf_dir)
547
+ ]
548
+
536
549
  def _init_area(self, input_params: dict) -> BaseGeometry:
537
550
  """Returns valid driver area for this process."""
538
551
  process_area = input_params["delimiters"]["effective_area"]
@@ -556,6 +569,9 @@ class InputData(base.InputData):
556
569
  return process_area
557
570
 
558
571
  def source_items(self) -> Generator[Item, None, None]:
572
+ if self.area.is_empty:
573
+ logger.debug("Input area is empty, nothing to be found here.")
574
+ return
559
575
  already_returned = set()
560
576
  for source in self.params.source:
561
577
  area = reproject_geometry(
@@ -25,7 +25,7 @@ from rasterio.profiles import Profile
25
25
  from rasterio.vrt import WarpedVRT
26
26
  from retry import retry
27
27
 
28
- from mapchete_eo.io.path import COMMON_RASTER_EXTENSIONS, asset_mpath, cached_path
28
+ from mapchete_eo.io.path import COMMON_RASTER_EXTENSIONS, asset_mpath
29
29
  from mapchete_eo.io.profiles import COGDeflateProfile
30
30
 
31
31
  logger = logging.getLogger(__name__)
@@ -43,7 +43,7 @@ class STACRasterBandProperties(BaseModel):
43
43
  nodataval: Optional[NodataVal] = None,
44
44
  ) -> STACRasterBandProperties:
45
45
  if asset.extra_fields.get("raster:offset", {}):
46
- properties = dict(
46
+ properties = dict( # pragma: no cover
47
47
  offset=asset.extra_fields.get("raster:offset"),
48
48
  scale=asset.extra_fields.get("raster:scale"),
49
49
  nodata=asset.extra_fields.get("nodata", nodataval),
@@ -394,9 +394,9 @@ def should_be_converted(
394
394
  return False
395
395
 
396
396
 
397
- def _read_vector_mask(mask_path):
397
+ def _read_vector_mask(mask_path: MPath):
398
398
  logger.debug("open %s with Fiona", mask_path)
399
- with cached_path(mask_path) as cached:
399
+ with mask_path.local_copy() as cached:
400
400
  try:
401
401
  with fiona_open(cached) as src:
402
402
  return list([dict(f) for f in src])
@@ -427,7 +427,7 @@ def read_mask_as_raster(
427
427
  if dst_grid:
428
428
  dst_grid = Grid.from_obj(dst_grid)
429
429
  try:
430
- with cached_path(path, active=cached_read) as path:
430
+ with path.local_copy(active=cached_read) as path:
431
431
  if path.suffix in COMMON_RASTER_EXTENSIONS:
432
432
  if dst_grid:
433
433
  array = read_raster_window(
@@ -41,6 +41,7 @@ def read_levelled_cube_to_np_array(
41
41
  out_dtype: DTypeLike = np.uint16,
42
42
  out_fill_value: NodataVal = 0,
43
43
  read_mask: Optional[np.ndarray] = None,
44
+ clear_caches: bool = True,
44
45
  ) -> ma.MaskedArray:
45
46
  """
46
47
  Read products as slices into a cube by filling up nodata gaps with next slice.
@@ -99,7 +100,7 @@ def read_levelled_cube_to_np_array(
99
100
  slices_read_count, slices_skip_count = 0, 0
100
101
 
101
102
  # pick slices one by one
102
- for slice_count, slice_ in enumerate(slices, 1):
103
+ for slice_count, slice in enumerate(slices, 1):
103
104
  # all filled up? let's get outta here!
104
105
  if not out.mask.any():
105
106
  logger.debug("cube has no pixels to be filled, quitting!")
@@ -113,11 +114,11 @@ def read_levelled_cube_to_np_array(
113
114
  logger.debug(
114
115
  "see if slice %s %s has some of the %s unmasked pixels for cube",
115
116
  slice_count,
116
- slice_,
117
+ slice,
117
118
  cube_nodata_mask.sum(),
118
119
  )
119
- with slice_.cached():
120
- slice_array = slice_.read(
120
+ with slice.cached(active=clear_caches):
121
+ slice_array = slice.read(
121
122
  merge_method=merge_method,
122
123
  product_read_kwargs=dict(
123
124
  product_read_kwargs,
@@ -133,12 +134,12 @@ def read_levelled_cube_to_np_array(
133
134
  )
134
135
  slices_read_count += 1
135
136
  except (EmptySliceException, CorruptedSlice) as exc:
136
- logger.debug("skipped slice %s: %s", slice_, str(exc))
137
+ logger.warning("skipped slice %s: %s", slice, str(exc))
137
138
  slices_skip_count += 1
138
139
  continue
139
140
 
140
141
  # if slice was not empty, fill pixels into cube
141
- logger.debug("add slice %s array to cube", slice_)
142
+ logger.debug("add slice %s array to cube", slice)
142
143
 
143
144
  # iterate through layers of cube
144
145
  for layer_index in range(target_height):
@@ -223,6 +224,7 @@ def read_levelled_cube_to_xarray(
223
224
  x_axis_name: str = "x",
224
225
  y_axis_name: str = "y",
225
226
  read_mask: Optional[np.ndarray] = None,
227
+ clear_caches: bool = True,
226
228
  ) -> xr.Dataset:
227
229
  """
228
230
  Read products as slices into a cube by filling up nodata gaps with next slice.
@@ -245,6 +247,7 @@ def read_levelled_cube_to_xarray(
245
247
  product_read_kwargs=product_read_kwargs,
246
248
  raise_empty=raise_empty,
247
249
  read_mask=read_mask,
250
+ clear_caches=clear_caches,
248
251
  ),
249
252
  slice_names=[f"layer-{ii}" for ii in range(target_height)],
250
253
  band_names=variables,
@@ -1,14 +1,11 @@
1
1
  import hashlib
2
2
  import logging
3
- from contextlib import contextmanager
4
3
  from enum import Enum
5
- from tempfile import TemporaryDirectory
6
- from typing import Generator, Tuple, Union
4
+ from typing import Tuple, Union
7
5
  from xml.etree.ElementTree import Element, fromstring
8
6
 
9
7
  import fsspec
10
8
  import pystac
11
- from mapchete.io import copy
12
9
  from mapchete.path import MPath
13
10
  from mapchete.settings import IORetrySettings
14
11
  from retry import retry
@@ -107,23 +104,6 @@ def path_in_paths(path, existing_paths) -> bool:
107
104
  return False
108
105
 
109
106
 
110
- @contextmanager
111
- @retry(logger=logger, **dict(IORetrySettings()))
112
- def cached_path(path: MPath, active: bool = True) -> Generator[MPath, None, None]:
113
- """If path is remote, download to temporary directory and return path."""
114
- if active and path.is_remote():
115
- with TemporaryDirectory() as tempdir:
116
- tempfile = MPath(tempdir) / path.name
117
- logger.debug("%s is remote, download to %s", path, tempfile)
118
- copy(
119
- path,
120
- tempfile,
121
- )
122
- yield tempfile
123
- else:
124
- yield path
125
-
126
-
127
107
  def asset_mpath(
128
108
  item: pystac.Item,
129
109
  asset: Union[str, Tuple[str, ...]],
@@ -51,6 +51,7 @@ def products_to_np_array(
51
51
  raise_empty: bool = True,
52
52
  out_dtype: Optional[DTypeLike] = None,
53
53
  read_mask: Optional[np.ndarray] = None,
54
+ clear_caches: bool = True,
54
55
  ) -> ma.MaskedArray:
55
56
  """Read grid window of EOProducts and merge into a 4D xarray."""
56
57
  return ma.stack(
@@ -69,6 +70,7 @@ def products_to_np_array(
69
70
  product_read_kwargs=product_read_kwargs,
70
71
  raise_empty=raise_empty,
71
72
  read_mask=read_mask,
73
+ clear_caches=clear_caches,
72
74
  )
73
75
  ]
74
76
  )
@@ -91,6 +93,7 @@ def products_to_xarray(
91
93
  raise_empty: bool = True,
92
94
  product_read_kwargs: dict = {},
93
95
  read_mask: Optional[np.ndarray] = None,
96
+ clear_caches: bool = True,
94
97
  ) -> xr.Dataset:
95
98
  """Read grid window of EOProducts and merge into a 4D xarray."""
96
99
  data_vars = [
@@ -108,6 +111,7 @@ def products_to_xarray(
108
111
  product_read_kwargs=product_read_kwargs,
109
112
  raise_empty=raise_empty,
110
113
  read_mask=read_mask,
114
+ clear_caches=clear_caches,
111
115
  )
112
116
  ]
113
117
  if merge_products_by and merge_products_by not in ["date", "datetime"]:
@@ -181,13 +185,20 @@ class Slice:
181
185
  return properties
182
186
 
183
187
  @contextmanager
184
- def cached(self) -> Generator[Slice, None, None]:
188
+ def cached(self, active: bool = True) -> Generator[Slice, None, None]:
185
189
  """Clear caches and run garbage collector when context manager is closed."""
186
190
  yield self
187
- with Timer() as tt:
188
- self.clear_cached_data()
189
- gc.collect()
190
- logger.debug("Slice cache cleared and garbage collected in %s", tt)
191
+ if active:
192
+ with Timer() as duration:
193
+ self.clear_cached_data()
194
+ # triggering the garbage collector helps fixing a memory leak on longer running
195
+ # process workers
196
+ cleared_objects = gc.collect()
197
+ logger.debug(
198
+ "Slice cache cleared and garbage collector cleared %s objects in %s",
199
+ cleared_objects,
200
+ duration,
201
+ )
191
202
 
192
203
  def clear_cached_data(self):
193
204
  logger.debug("clear caches of all products in slice")
@@ -360,6 +371,7 @@ def generate_slice_dataarrays(
360
371
  product_read_kwargs: dict = {},
361
372
  raise_empty: bool = True,
362
373
  read_mask: Optional[np.ndarray] = None,
374
+ clear_caches: bool = True,
363
375
  ) -> Iterator[xr.DataArray]:
364
376
  """
365
377
  Yield products or merged products into slices as DataArrays.
@@ -392,7 +404,7 @@ def generate_slice_dataarrays(
392
404
  try:
393
405
  # if merge_products_by is none, each slice contains just one product
394
406
  # so nothing will have to be merged anyways
395
- with slice.cached():
407
+ with slice.cached(active=clear_caches):
396
408
  yield to_dataarray(
397
409
  merge_products(
398
410
  products=slice.products,
@@ -416,8 +428,8 @@ def generate_slice_dataarrays(
416
428
  )
417
429
  # if at least one slice can be yielded, the stack is not empty
418
430
  stack_empty = False
419
- except (EmptySliceException, CorruptedSlice) as exception:
420
- logger.warning(exception)
431
+ except (EmptySliceException, CorruptedSlice) as exc:
432
+ logger.warning("skipped slice %s: %s", slice, str(exc))
421
433
 
422
434
  if stack_empty:
423
435
  raise EmptyStackException("all slices are empty")
@@ -60,6 +60,7 @@ def masks_to_xarray(
60
60
  sort: Optional[SortMethodConfig] = None,
61
61
  raise_empty: bool = True,
62
62
  product_read_kwargs: dict = {},
63
+ clear_caches: bool = True,
63
64
  ) -> xr.Dataset:
64
65
  """
65
66
  Read masks of products and merge into an xarray.Dataset.
@@ -74,6 +75,7 @@ def masks_to_xarray(
74
75
  sort=sort,
75
76
  product_read_kwargs=product_read_kwargs,
76
77
  raise_empty=raise_empty,
78
+ clear_caches=clear_caches,
77
79
  )
78
80
  ]
79
81
  if merge_products_by and merge_products_by not in ["date", "datetime"]:
@@ -256,6 +258,7 @@ def generate_slice_masks_dataarrays(
256
258
  mask_name: List = ["EOxCloudless_masks"],
257
259
  product_read_kwargs: dict = {},
258
260
  raise_empty: bool = True,
261
+ clear_caches: bool = True,
259
262
  ) -> Iterator[xr.DataArray]:
260
263
  """
261
264
  Yield products or merged products into slices as DataArrays.
@@ -279,7 +282,7 @@ def generate_slice_masks_dataarrays(
279
282
  try:
280
283
  # if merge_products_by is none, each slice contains just one product
281
284
  # so nothing will have to be merged anyways
282
- with slice.cached():
285
+ with slice.cached(active=clear_caches):
283
286
  yield to_dataarray(
284
287
  merge_products_masks(
285
288
  products=slice.products,
@@ -297,8 +300,8 @@ def generate_slice_masks_dataarrays(
297
300
  )
298
301
  # if at least one slice can be yielded, the stack is not empty
299
302
  stack_empty = False
300
- except (EmptySliceException, CorruptedSlice):
301
- pass
303
+ except (EmptySliceException, CorruptedSlice) as exc:
304
+ logger.warning("skipped slice %s: %s", slice, str(exc))
302
305
 
303
306
  if stack_empty:
304
307
  raise EmptyStackException("all slices are empty")
@@ -140,7 +140,7 @@ def map_cdse_paths_to_jp2_archive(item: Item) -> Item:
140
140
  elif asset_name == "granule_metadata":
141
141
  asset.href = str(product_basepath / "metadata.xml")
142
142
  # change band asset names and point to their new locations
143
- elif asset_name in CDSE_ASSET_NAME_MAPPING:
143
+ elif asset_name in CDSE_ASSET_NAME_MAPPING: # pragma: no cover
144
144
  name, resolution = asset_name.split("_")
145
145
  asset.href = product_basepath / f"R{resolution}" / f"{name}.jp2"
146
146
  asset_name = CDSE_ASSET_NAME_MAPPING[asset_name]
@@ -155,7 +155,7 @@ def map_cdse_paths_to_jp2_archive(item: Item) -> Item:
155
155
 
156
156
  @creates_s2metadata(from_collections=["CDSE"], to_metadata_archives=["CDSE"])
157
157
  def cdse_s2metadata(item: Item) -> S2Metadata:
158
- return S2Metadata.from_stac_item(
158
+ return S2Metadata.from_stac_item( # pragma: no cover
159
159
  item,
160
160
  path_mapper=CDSEPathMapper(MPath(item.assets["granule_metadata"].href)),
161
161
  processing_baseline_field="processing:version",
@@ -201,7 +201,8 @@ class S2Product(EOProduct, EOProductProtocol):
201
201
 
202
202
  @property
203
203
  def item(self) -> Item:
204
- if not self._item:
204
+ if self._item is None:
205
+ logger.debug("%s no cached STAC Item found, recreating ...", self.id)
205
206
  if self._item_dict:
206
207
  self._item = Item.from_dict(self._item_dict)
207
208
  else:
@@ -213,7 +214,8 @@ class S2Product(EOProduct, EOProductProtocol):
213
214
 
214
215
  @property
215
216
  def metadata(self) -> S2Metadata:
216
- if not self._metadata:
217
+ if self._metadata is None:
218
+ logger.debug("%s no cached metadata found, recreating ...", self.id)
217
219
  if self._metadata_mapper:
218
220
  self._metadata = self._metadata_mapper(self.item)
219
221
  else:
@@ -36,7 +36,7 @@ class STACSearchCollection(StaticCollectionWriterMixin, CollectionSearcher):
36
36
  def eo_bands(self) -> List[str]:
37
37
  item_assets = self.client.extra_fields.get("item_assets", {})
38
38
  for v in item_assets.values():
39
- if "eo:bands" in v and "data" in v.get("roles", []):
39
+ if "eo:bands" in v and "data" in v.get("roles", []): # pragma: no cover
40
40
  return ["eo:bands"]
41
41
  else: # pragma: no cover
42
42
  logger.debug("cannot find eo:bands definition from collections")
@@ -48,9 +48,11 @@ docs = [
48
48
  "myst-parser",
49
49
  ]
50
50
  test = [
51
+ "httpx",
51
52
  "pytest",
52
- "pytest-coverage",
53
- "pytest-lazy-fixtures"
53
+ "pytest-cov",
54
+ "pytest-lazy-fixtures",
55
+ "pytest-split"
54
56
  ]
55
57
 
56
58
  [project.entry-points."mapchete.cli.commands"]
@@ -81,10 +83,20 @@ ignore_missing_imports = true
81
83
 
82
84
  [tool.coverage.run]
83
85
  branch = true
86
+ relative_files = true
84
87
  source = ["mapchete_eo"]
85
88
 
86
89
  [tool.pytest.ini_options]
87
- addopts = "--durations 20 --verbose --nf --cov=mapchete_eo --cov-report=term-missing:skip-covered"
90
+ addopts = [
91
+ "--durations=20",
92
+ "--verbose",
93
+ "--splitting-algorithm", "least_duration",
94
+ "--durations-path", "tests/.test_duration",
95
+ "--nf",
96
+ "--cov=mapchete_eo",
97
+ "--cov-report=term-missing:skip-covered",
98
+ "--cov-append"
99
+ ]
88
100
  testpaths = ["tests"]
89
101
  markers = [
90
102
  "remote: marks tests which require acces to remote resources (deselect with '-m \"not remote\"')",
@@ -93,5 +105,6 @@ markers = [
93
105
 
94
106
  [tool.pyproject2conda]
95
107
  channels = ["conda-forge"]
96
- # Maps PyPI names to Conda-Forge names if they differ
97
- map_deps = { "opencv-python-headless" = "opencv", "pystac-client" = "python-pystac-client" }
108
+
109
+ [tool.pyproject2conda.dependencies]
110
+ opencv-python-headless = { skip = true, packages = "opencv" }
@@ -1 +0,0 @@
1
- __version__ = "2026.2.0"
File without changes