mapchete 2026.2.1__tar.gz → 2026.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {mapchete-2026.2.1 → mapchete-2026.2.2}/.gitignore +2 -1
- {mapchete-2026.2.1 → mapchete-2026.2.2}/PKG-INFO +3 -1
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/__init__.py +1 -1
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/errors.py +42 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/formats/tools.py +4 -1
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/index.py +1 -1
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/io/raster/__init__.py +6 -1
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/io/raster/read.py +21 -12
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/io/raster/referenced_raster.py +76 -1
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/io/vector/read.py +5 -4
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/processes/contours.py +3 -1
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/settings.py +13 -2
- {mapchete-2026.2.1 → mapchete-2026.2.2}/pyproject.toml +13 -1
- {mapchete-2026.2.1 → mapchete-2026.2.2}/LICENSE +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/README.rst +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/_deprecated.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/bounds.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/cli/__init__.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/cli/mapchete/__init__.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/cli/mapchete/convert.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/cli/mapchete/cp.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/cli/mapchete/create.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/cli/mapchete/execute.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/cli/mapchete/formats.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/cli/mapchete/index.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/cli/mapchete/processes.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/cli/mapchete/rm.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/cli/mapchete/serve.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/cli/mapchete/show_env.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/cli/mapchete/stac.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/cli/mpath/__init__.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/cli/mpath/cp.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/cli/mpath/exists.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/cli/mpath/ls.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/cli/mpath/read_json.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/cli/mpath/read_text.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/cli/mpath/read_yaml.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/cli/mpath/rm.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/cli/mpath/sync.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/cli/options.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/cli/progress_bar.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/commands/__init__.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/commands/convert.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/commands/cp.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/commands/execute.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/commands/index.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/commands/observer.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/commands/parser.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/commands/rm.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/config/__init__.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/config/base.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/config/models.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/config/parse.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/config/process_func.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/enums.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/executor/__init__.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/executor/base.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/executor/concurrent_futures.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/executor/dask.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/executor/future.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/executor/sequential.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/executor/types.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/formats/__init__.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/formats/base.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/formats/default/__init__.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/formats/default/_fiona_base.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/formats/default/flatgeobuf.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/formats/default/geojson.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/formats/default/gtiff.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/formats/default/mapchete_input.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/formats/default/png.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/formats/default/png_hillshade.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/formats/default/raster_file.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/formats/default/tile_directory.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/formats/default/vector_file.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/formats/drivers.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/formats/loaders.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/formats/protocols.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/geometry/__init__.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/geometry/clip.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/geometry/filter.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/geometry/footprints.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/geometry/latlon.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/geometry/repair.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/geometry/reproject.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/geometry/segmentize.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/geometry/shape.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/geometry/transform.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/geometry/types.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/grid.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/io/__init__.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/io/_json.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/io/_misc.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/io/_path.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/io/profiles.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/io/raster/array.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/io/raster/convert.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/io/raster/mosaic.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/io/raster/open.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/io/raster/write.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/io/vector/__init__.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/io/vector/convert.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/io/vector/indexed_features.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/io/vector/open.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/io/vector/types.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/io/vector/write.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/log.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/path.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/pretty.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/process_func_special_types.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/processes/__init__.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/processes/clip.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/processes/convert.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/processes/examples/__init__.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/processes/examples/example_process.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/processes/hillshade.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/processing/__init__.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/processing/base.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/processing/execute.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/processing/mp.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/processing/profilers/__init__.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/processing/profilers/memory.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/processing/profilers/requests.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/processing/profilers/time.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/processing/tasks.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/processing/types.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/protocols.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/registered.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/stac/__init__.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/stac/models.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/stac/tiled_assets.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/static/__init__.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/static/index.html +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/static/mapchete_template.mapchete +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/static/process_template.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/testing.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/tile.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/timer.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/types.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/validate.py +0 -0
- {mapchete-2026.2.1 → mapchete-2026.2.2}/mapchete/zoom_levels.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: mapchete
|
|
3
|
-
Version: 2026.2.
|
|
3
|
+
Version: 2026.2.2
|
|
4
4
|
Summary: Tile-based geodata processing using rasterio & Fiona
|
|
5
5
|
Project-URL: Homepage, https://github.com/ungarj/mapchete
|
|
6
6
|
Author-email: Joachim Ungar <joachim.ungar@gmail.com>
|
|
@@ -89,6 +89,8 @@ Requires-Dist: pytest-cov>=2.8.1; extra == 'test'
|
|
|
89
89
|
Requires-Dist: pytest-env; extra == 'test'
|
|
90
90
|
Requires-Dist: pytest-flask>=1.0.0; extra == 'test'
|
|
91
91
|
Requires-Dist: pytest-lazy-fixtures; extra == 'test'
|
|
92
|
+
Requires-Dist: pytest-mock; extra == 'test'
|
|
93
|
+
Requires-Dist: pytest-split; extra == 'test'
|
|
92
94
|
Requires-Dist: pytest<8,>=5.4.1; extra == 'test'
|
|
93
95
|
Requires-Dist: rio-cogeo>=5.0.0; extra == 'test'
|
|
94
96
|
Provides-Extra: vrt
|
|
@@ -67,3 +67,45 @@ class NoGeoError(AttributeError):
|
|
|
67
67
|
|
|
68
68
|
class NoCRSError(AttributeError):
|
|
69
69
|
"""Raised when object does not contain a CRS."""
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _is_frozen_exc(exc: Exception) -> bool:
|
|
73
|
+
"""Return True if the exception type does not support attribute assignment.
|
|
74
|
+
|
|
75
|
+
Cython extension types (e.g. rasterio._err.CPLE_AppDefinedError) typically
|
|
76
|
+
raise AttributeError when you try to set arbitrary attributes, which causes
|
|
77
|
+
tblib/dask unpickling to fail in distributed environments.
|
|
78
|
+
"""
|
|
79
|
+
_watcher = "_clean_exception_probe"
|
|
80
|
+
try:
|
|
81
|
+
setattr(exc, _watcher, None)
|
|
82
|
+
delattr(exc, _watcher)
|
|
83
|
+
return False
|
|
84
|
+
except (AttributeError, TypeError):
|
|
85
|
+
return True
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def clean_exception(exc: Exception) -> Exception:
|
|
89
|
+
"""Sanitize an exception chain so it can be safely pickled by tblib/dask.
|
|
90
|
+
|
|
91
|
+
Replaces any exception whose type does not support attribute assignment
|
|
92
|
+
(e.g. Cython extension types) with an equivalent ``RuntimeError``, then
|
|
93
|
+
recurses into ``__cause__`` and ``__context__`` to clean the full chain.
|
|
94
|
+
"""
|
|
95
|
+
if exc is None:
|
|
96
|
+
return None
|
|
97
|
+
|
|
98
|
+
try:
|
|
99
|
+
safe_exc = (
|
|
100
|
+
RuntimeError(f"{type(exc).__name__}: {exc}") if _is_frozen_exc(exc) else exc
|
|
101
|
+
)
|
|
102
|
+
for attr in ("__cause__", "__context__"):
|
|
103
|
+
if (chained := getattr(exc, attr, None)) is not None:
|
|
104
|
+
try:
|
|
105
|
+
setattr(safe_exc, attr, clean_exception(chained))
|
|
106
|
+
except AttributeError:
|
|
107
|
+
pass
|
|
108
|
+
return safe_exc
|
|
109
|
+
except Exception:
|
|
110
|
+
# last-resort fallback if reconstruction itself fails
|
|
111
|
+
return Exception(str(exc))
|
|
@@ -15,7 +15,7 @@ from rasterio.crs import CRS
|
|
|
15
15
|
from shapely.geometry.base import BaseGeometry
|
|
16
16
|
|
|
17
17
|
from mapchete.errors import MapcheteConfigError, MapcheteDriverError
|
|
18
|
-
from mapchete.
|
|
18
|
+
from mapchete.path import MPath
|
|
19
19
|
from mapchete.registered import drivers
|
|
20
20
|
from mapchete.tile import BufferedTilePyramid
|
|
21
21
|
from mapchete.types import MPathLike
|
|
@@ -112,6 +112,9 @@ def driver_from_file(input_file: MPathLike, quick: bool = True) -> str:
|
|
|
112
112
|
pass
|
|
113
113
|
|
|
114
114
|
# brute force by trying to open file with rasterio and fiona:
|
|
115
|
+
from mapchete.io.vector import fiona_open
|
|
116
|
+
from mapchete.io.raster import rasterio_open
|
|
117
|
+
|
|
115
118
|
try:
|
|
116
119
|
logger.debug("try to open %s with rasterio...", input_file)
|
|
117
120
|
with rasterio_open(input_file): # pragma: no cover
|
|
@@ -17,7 +17,11 @@ from mapchete.io.raster.read import (
|
|
|
17
17
|
read_raster_window,
|
|
18
18
|
tiles_to_affine_shape,
|
|
19
19
|
)
|
|
20
|
-
from mapchete.io.raster.referenced_raster import
|
|
20
|
+
from mapchete.io.raster.referenced_raster import (
|
|
21
|
+
ReferencedRaster,
|
|
22
|
+
ReferencedRasterInput,
|
|
23
|
+
read_raster,
|
|
24
|
+
)
|
|
21
25
|
from mapchete.io.raster.write import rasterio_write, write_raster_window
|
|
22
26
|
|
|
23
27
|
__all__ = [
|
|
@@ -37,6 +41,7 @@ __all__ = [
|
|
|
37
41
|
"tiles_to_affine_shape",
|
|
38
42
|
"memory_file",
|
|
39
43
|
"ReferencedRaster",
|
|
44
|
+
"ReferencedRasterInput",
|
|
40
45
|
"read_raster",
|
|
41
46
|
"rasterio_write",
|
|
42
47
|
"write_raster_window",
|
|
@@ -22,7 +22,7 @@ from rasterio.warp import reproject
|
|
|
22
22
|
from retry import retry
|
|
23
23
|
from tilematrix import Shape
|
|
24
24
|
|
|
25
|
-
from mapchete.errors import MapcheteIOError
|
|
25
|
+
from mapchete.errors import MapcheteIOError, clean_exception
|
|
26
26
|
from mapchete.geometry.clip import clip_geometry_to_pyramid_bounds
|
|
27
27
|
from mapchete.grid import Grid
|
|
28
28
|
from mapchete.io.raster.array import extract_from_array, prepare_masked_array
|
|
@@ -201,9 +201,10 @@ def _read_raster_window(
|
|
|
201
201
|
raise
|
|
202
202
|
except Exception as exception: # pragma: no cover
|
|
203
203
|
_, _, exc_traceback = sys.exc_info()
|
|
204
|
+
clean_exc = clean_exception(exception)
|
|
204
205
|
raise MapcheteIOError(
|
|
205
|
-
f"failed to read {input_file} due to a {str(
|
|
206
|
-
).with_traceback(exc_traceback) from
|
|
206
|
+
f"failed to read {input_file} due to a {str(clean_exc)}"
|
|
207
|
+
).with_traceback(exc_traceback) from clean_exc
|
|
207
208
|
|
|
208
209
|
|
|
209
210
|
def _get_warped_edge_array(
|
|
@@ -356,6 +357,16 @@ def _rasterio_read(
|
|
|
356
357
|
|
|
357
358
|
|
|
358
359
|
@retry(logger=logger, **dict(IORetrySettings()))
|
|
360
|
+
def _read_raster_no_crs(
|
|
361
|
+
input_file: MPathLike, indexes: Optional[Union[int, List[int]]] = None, **kwargs
|
|
362
|
+
) -> ma.MaskedArray:
|
|
363
|
+
"""Retries the raw read."""
|
|
364
|
+
with warnings.catch_warnings():
|
|
365
|
+
warnings.simplefilter("ignore")
|
|
366
|
+
with rasterio_read(input_file, "r") as src:
|
|
367
|
+
return src.read(indexes=indexes, masked=True)
|
|
368
|
+
|
|
369
|
+
|
|
359
370
|
def read_raster_no_crs(
|
|
360
371
|
input_file: MPathLike, indexes: Optional[Union[int, List[int]]] = None, **kwargs
|
|
361
372
|
) -> ma.MaskedArray:
|
|
@@ -365,16 +376,14 @@ def read_raster_no_crs(
|
|
|
365
376
|
Raises
|
|
366
377
|
------
|
|
367
378
|
FileNotFoundError if file cannot be found.
|
|
379
|
+
MapcheteIOError on other failures.
|
|
368
380
|
"""
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
raise
|
|
376
|
-
except Exception as exc:
|
|
377
|
-
raise MapcheteIOError(exc)
|
|
381
|
+
try:
|
|
382
|
+
return _read_raster_no_crs(input_file, indexes=indexes, **kwargs)
|
|
383
|
+
except FileNotFoundError:
|
|
384
|
+
raise
|
|
385
|
+
except Exception as exc:
|
|
386
|
+
raise MapcheteIOError(str(clean_exc := clean_exception(exc))) from clean_exc
|
|
378
387
|
|
|
379
388
|
|
|
380
389
|
def _extract_filenotfound_exception(rio_exc: Exception, path: MPath):
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
import logging
|
|
4
|
-
from typing import List, Optional, Tuple, Union
|
|
4
|
+
from typing import List, Optional, Tuple, Union, Any
|
|
5
|
+
import uuid
|
|
5
6
|
|
|
6
7
|
import numpy as np
|
|
7
8
|
import numpy.ma as ma
|
|
@@ -11,6 +12,7 @@ from retry import retry
|
|
|
11
12
|
from shapely.geometry import mapping, shape
|
|
12
13
|
|
|
13
14
|
from mapchete.bounds import Bounds
|
|
15
|
+
from mapchete.formats.protocols import RasterInput
|
|
14
16
|
from mapchete.grid import Grid
|
|
15
17
|
from mapchete.io.raster.array import prepare_array, resample_from_array
|
|
16
18
|
from mapchete.io.raster.open import rasterio_open
|
|
@@ -18,6 +20,7 @@ from mapchete.io.raster.read import read_raster_window
|
|
|
18
20
|
from mapchete.path import MPath
|
|
19
21
|
from mapchete.protocols import GridProtocol
|
|
20
22
|
from mapchete.settings import IORetrySettings
|
|
23
|
+
from mapchete.tile import BufferedTile
|
|
21
24
|
from mapchete.types import BoundsLike, CRSLike, MPathLike, NodataVal
|
|
22
25
|
|
|
23
26
|
logger = logging.getLogger(__name__)
|
|
@@ -235,3 +238,75 @@ def read_raster(
|
|
|
235
238
|
**kwargs,
|
|
236
239
|
) -> ReferencedRaster:
|
|
237
240
|
return ReferencedRaster.from_file(inp, grid=grid, masked=masked, **kwargs)
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
class ReferencedRasterInput(RasterInput):
|
|
244
|
+
"""Wrapper around ReferencedRaster so outputs from execute functions can be passed around."""
|
|
245
|
+
|
|
246
|
+
referenced_raster: ReferencedRaster
|
|
247
|
+
preprocessing_tasks_results: dict
|
|
248
|
+
input_key: str
|
|
249
|
+
tile: BufferedTile
|
|
250
|
+
|
|
251
|
+
def __init__(
|
|
252
|
+
self,
|
|
253
|
+
referenced_raster: ReferencedRaster,
|
|
254
|
+
tile: BufferedTile,
|
|
255
|
+
input_key: Optional[str] = None,
|
|
256
|
+
preprocessing_tasks_results: Optional[dict] = None,
|
|
257
|
+
):
|
|
258
|
+
self.referenced_raster = referenced_raster
|
|
259
|
+
self.tile = tile
|
|
260
|
+
self.input_key = input_key or uuid.uuid4().hex
|
|
261
|
+
preprocessing_tasks_results = preprocessing_tasks_results or dict()
|
|
262
|
+
|
|
263
|
+
def read(
|
|
264
|
+
self,
|
|
265
|
+
indexes: Optional[Union[int, List[int]]] = None,
|
|
266
|
+
resampling: str = "nearest",
|
|
267
|
+
**__,
|
|
268
|
+
) -> np.ndarray:
|
|
269
|
+
"""Either read full array or resampled to grid."""
|
|
270
|
+
return self.referenced_raster.read(indexes=indexes, resampling=resampling)
|
|
271
|
+
|
|
272
|
+
def is_empty(self) -> bool:
|
|
273
|
+
"""Checks if input is empty here."""
|
|
274
|
+
return self.referenced_raster.masked_array().mask.all()
|
|
275
|
+
|
|
276
|
+
def set_preprocessing_task_result(
|
|
277
|
+
self, task_key: str, result: Any
|
|
278
|
+
) -> None: # pragma: no cover
|
|
279
|
+
# should this even be called?
|
|
280
|
+
pass
|
|
281
|
+
|
|
282
|
+
def __enter__(self) -> ReferencedRasterInput: # pragma: no cover
|
|
283
|
+
"""Required for 'with' statement."""
|
|
284
|
+
return self
|
|
285
|
+
|
|
286
|
+
def __exit__(self, *args):
|
|
287
|
+
"""Clean up."""
|
|
288
|
+
|
|
289
|
+
@staticmethod
|
|
290
|
+
def from_array(
|
|
291
|
+
array: np.ndarray, tile: BufferedTile, **kwargs
|
|
292
|
+
) -> ReferencedRasterInput:
|
|
293
|
+
return ReferencedRasterInput(
|
|
294
|
+
ReferencedRaster.from_array_like(
|
|
295
|
+
array, transform=tile.transform, crs=tile.crs
|
|
296
|
+
),
|
|
297
|
+
tile=tile,
|
|
298
|
+
**kwargs,
|
|
299
|
+
)
|
|
300
|
+
|
|
301
|
+
@staticmethod
|
|
302
|
+
def from_file(
|
|
303
|
+
path: MPathLike,
|
|
304
|
+
tile: BufferedTile,
|
|
305
|
+
masked: bool = True,
|
|
306
|
+
**kwargs,
|
|
307
|
+
) -> ReferencedRasterInput:
|
|
308
|
+
return ReferencedRasterInput(
|
|
309
|
+
ReferencedRaster.from_file(path, grid=tile, masked=masked),
|
|
310
|
+
tile=tile,
|
|
311
|
+
**kwargs,
|
|
312
|
+
)
|
|
@@ -12,7 +12,7 @@ from shapely import clip_by_rect
|
|
|
12
12
|
from shapely.errors import TopologicalError
|
|
13
13
|
from shapely.geometry import mapping
|
|
14
14
|
|
|
15
|
-
from mapchete.errors import MapcheteIOError
|
|
15
|
+
from mapchete.errors import MapcheteIOError, clean_exception
|
|
16
16
|
from mapchete.geometry import (
|
|
17
17
|
filter_by_geometry_type,
|
|
18
18
|
multipart_to_singleparts,
|
|
@@ -52,7 +52,6 @@ def fiona_read(
|
|
|
52
52
|
Wrapper around fiona.open but fiona.Env is set according to path properties.
|
|
53
53
|
"""
|
|
54
54
|
path = MPath.from_inp(path)
|
|
55
|
-
|
|
56
55
|
try:
|
|
57
56
|
with path.fio_env() as env:
|
|
58
57
|
logger.debug("reading %s with GDAL options %s", str(path), env.options)
|
|
@@ -111,9 +110,10 @@ def read_vector_window_generator(
|
|
|
111
110
|
raise
|
|
112
111
|
except Exception as exception: # pragma: no cover
|
|
113
112
|
_, _, exc_traceback = sys.exc_info()
|
|
113
|
+
clean_exc = clean_exception(exception)
|
|
114
114
|
raise MapcheteIOError(
|
|
115
|
-
f"failed to read {inp} due to a {str(
|
|
116
|
-
).with_traceback(exc_traceback) from
|
|
115
|
+
f"failed to read {inp} due to a {str(clean_exc)}"
|
|
116
|
+
).with_traceback(exc_traceback) from clean_exc
|
|
117
117
|
|
|
118
118
|
|
|
119
119
|
def read_vector_window(
|
|
@@ -195,6 +195,7 @@ def _read_vector_window_from_file(
|
|
|
195
195
|
raise IOError(f"failed to read {inp}") from exc
|
|
196
196
|
|
|
197
197
|
|
|
198
|
+
# Warning: This retry wont trigger as yield does not have materilized object/function
|
|
198
199
|
@retry(
|
|
199
200
|
logger=logger,
|
|
200
201
|
**IORetrySettings().model_dump(exclude_none=True),
|
|
@@ -95,6 +95,8 @@ def execute(
|
|
|
95
95
|
matching_precision=td_matching_precision,
|
|
96
96
|
fallback_to_higher_zoom=td_fallback_to_higher_zoom,
|
|
97
97
|
)
|
|
98
|
+
if dem_data.ndim == 3:
|
|
99
|
+
dem_data = dem_data[0]
|
|
98
100
|
if dem_data.mask.all(): # pragma: no cover
|
|
99
101
|
logger.debug("DEM data empty over tile")
|
|
100
102
|
raise Empty("DEM data empty over tile")
|
|
@@ -153,7 +155,7 @@ def generate_contours(
|
|
|
153
155
|
properties={field: elevation},
|
|
154
156
|
geometry=mapping(path_to_linestring(path, tile)),
|
|
155
157
|
)
|
|
156
|
-
except GEOSException:
|
|
158
|
+
except GEOSException: # pragma: no cover
|
|
157
159
|
continue
|
|
158
160
|
|
|
159
161
|
|
|
@@ -9,9 +9,10 @@ from aiohttp import ClientPayloadError, ClientResponseError
|
|
|
9
9
|
from aiohttp.client_exceptions import ServerDisconnectedError
|
|
10
10
|
from fiona.errors import FionaError
|
|
11
11
|
from fsspec.exceptions import FSTimeoutError
|
|
12
|
-
from pydantic import NonNegativeFloat, NonNegativeInt
|
|
12
|
+
from pydantic import NonNegativeFloat, NonNegativeInt, model_validator
|
|
13
13
|
from pydantic_settings import BaseSettings, SettingsConfigDict
|
|
14
|
-
from rasterio.errors import RasterioError
|
|
14
|
+
from rasterio.errors import RasterioError, RasterioIOError
|
|
15
|
+
from rasterio._err import CPLE_AppDefinedError, CPLE_OpenFailedError, CPLE_FileIOError
|
|
15
16
|
|
|
16
17
|
from mapchete.executor import Concurrency
|
|
17
18
|
|
|
@@ -66,16 +67,26 @@ class IORetrySettings(BaseSettings):
|
|
|
66
67
|
SystemError,
|
|
67
68
|
TimeoutError,
|
|
68
69
|
RasterioError,
|
|
70
|
+
RasterioIOError,
|
|
69
71
|
FionaError,
|
|
70
72
|
FSTimeoutError,
|
|
71
73
|
ServerDisconnectedError,
|
|
72
74
|
ClientResponseError,
|
|
73
75
|
ClientPayloadError,
|
|
76
|
+
CPLE_AppDefinedError,
|
|
77
|
+
CPLE_OpenFailedError,
|
|
78
|
+
CPLE_FileIOError,
|
|
74
79
|
)
|
|
75
80
|
|
|
76
81
|
# read from environment
|
|
77
82
|
model_config = SettingsConfigDict(env_prefix="MAPCHETE_IO_RETRY_")
|
|
78
83
|
|
|
84
|
+
@model_validator(mode="after")
|
|
85
|
+
def check_backoff(self) -> "IORetrySettings":
|
|
86
|
+
if self.delay > 0 and self.backoff == 0:
|
|
87
|
+
self.backoff = 1.0
|
|
88
|
+
return self
|
|
89
|
+
|
|
79
90
|
|
|
80
91
|
class MapcheteOptions(BaseSettings):
|
|
81
92
|
# timeout granted when fetching future results or exceptions
|
|
@@ -111,6 +111,8 @@ test = [
|
|
|
111
111
|
"pytest-env",
|
|
112
112
|
"pytest-flask>=1.0.0",
|
|
113
113
|
"pytest-lazy-fixtures",
|
|
114
|
+
"pytest-mock",
|
|
115
|
+
"pytest-split",
|
|
114
116
|
"rio-cogeo>=5.0.0",
|
|
115
117
|
]
|
|
116
118
|
|
|
@@ -188,7 +190,17 @@ env = [
|
|
|
188
190
|
"D:AK2=bar",
|
|
189
191
|
"D:FILE1=dummy2.tif"
|
|
190
192
|
]
|
|
191
|
-
addopts =
|
|
193
|
+
addopts = [
|
|
194
|
+
"--durations=20",
|
|
195
|
+
"--verbose",
|
|
196
|
+
"--splitting-algorithm", "least_duration",
|
|
197
|
+
"--durations-path", "test/.test_durations",
|
|
198
|
+
"--nf",
|
|
199
|
+
"--cov=mapchete",
|
|
200
|
+
"--cov-report=term-missing:skip-covered",
|
|
201
|
+
"--cov-append"
|
|
202
|
+
]
|
|
203
|
+
testpaths = ["test"]
|
|
192
204
|
|
|
193
205
|
[tool.unidep]
|
|
194
206
|
channels = ["conda-forge"]
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|