mapchete 2026.2.2__tar.gz → 2026.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {mapchete-2026.2.2 → mapchete-2026.3.0}/PKG-INFO +2 -4
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/__init__.py +1 -1
- mapchete-2026.3.0/mapchete/cli/mapchete/rm.py +59 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/cli/progress_bar.py +3 -1
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/commands/rm.py +76 -41
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/config/base.py +12 -5
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/formats/default/_fiona_base.py +1 -1
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/formats/default/gtiff.py +1 -1
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/formats/default/raster_file.py +1 -1
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/formats/default/tile_directory.py +1 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/formats/default/vector_file.py +1 -1
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/formats/protocols.py +1 -1
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/io/raster/referenced_raster.py +1 -1
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/io/vector/indexed_features.py +122 -56
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/path.py +26 -13
- {mapchete-2026.2.2 → mapchete-2026.3.0}/pyproject.toml +1 -4
- mapchete-2026.2.2/mapchete/cli/mapchete/rm.py +0 -49
- {mapchete-2026.2.2 → mapchete-2026.3.0}/.gitignore +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/LICENSE +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/README.rst +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/_deprecated.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/bounds.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/cli/__init__.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/cli/mapchete/__init__.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/cli/mapchete/convert.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/cli/mapchete/cp.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/cli/mapchete/create.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/cli/mapchete/execute.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/cli/mapchete/formats.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/cli/mapchete/index.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/cli/mapchete/processes.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/cli/mapchete/serve.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/cli/mapchete/show_env.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/cli/mapchete/stac.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/cli/mpath/__init__.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/cli/mpath/cp.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/cli/mpath/exists.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/cli/mpath/ls.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/cli/mpath/read_json.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/cli/mpath/read_text.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/cli/mpath/read_yaml.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/cli/mpath/rm.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/cli/mpath/sync.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/cli/options.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/commands/__init__.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/commands/convert.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/commands/cp.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/commands/execute.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/commands/index.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/commands/observer.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/commands/parser.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/config/__init__.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/config/models.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/config/parse.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/config/process_func.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/enums.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/errors.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/executor/__init__.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/executor/base.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/executor/concurrent_futures.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/executor/dask.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/executor/future.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/executor/sequential.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/executor/types.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/formats/__init__.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/formats/base.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/formats/default/__init__.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/formats/default/flatgeobuf.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/formats/default/geojson.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/formats/default/mapchete_input.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/formats/default/png.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/formats/default/png_hillshade.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/formats/drivers.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/formats/loaders.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/formats/tools.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/geometry/__init__.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/geometry/clip.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/geometry/filter.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/geometry/footprints.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/geometry/latlon.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/geometry/repair.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/geometry/reproject.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/geometry/segmentize.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/geometry/shape.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/geometry/transform.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/geometry/types.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/grid.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/index.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/io/__init__.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/io/_json.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/io/_misc.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/io/_path.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/io/profiles.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/io/raster/__init__.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/io/raster/array.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/io/raster/convert.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/io/raster/mosaic.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/io/raster/open.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/io/raster/read.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/io/raster/write.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/io/vector/__init__.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/io/vector/convert.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/io/vector/open.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/io/vector/read.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/io/vector/types.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/io/vector/write.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/log.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/pretty.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/process_func_special_types.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/processes/__init__.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/processes/clip.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/processes/contours.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/processes/convert.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/processes/examples/__init__.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/processes/examples/example_process.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/processes/hillshade.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/processing/__init__.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/processing/base.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/processing/execute.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/processing/mp.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/processing/profilers/__init__.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/processing/profilers/memory.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/processing/profilers/requests.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/processing/profilers/time.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/processing/tasks.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/processing/types.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/protocols.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/registered.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/settings.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/stac/__init__.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/stac/models.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/stac/tiled_assets.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/static/__init__.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/static/index.html +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/static/mapchete_template.mapchete +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/static/process_template.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/testing.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/tile.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/timer.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/types.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/validate.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/zoom_levels.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: mapchete
|
|
3
|
-
Version: 2026.
|
|
3
|
+
Version: 2026.3.0
|
|
4
4
|
Summary: Tile-based geodata processing using rasterio & Fiona
|
|
5
5
|
Project-URL: Homepage, https://github.com/ungarj/mapchete
|
|
6
6
|
Author-email: Joachim Ungar <joachim.ungar@gmail.com>
|
|
@@ -54,7 +54,6 @@ Requires-Dist: matplotlib<3.11; extra == 'complete'
|
|
|
54
54
|
Requires-Dist: memray; extra == 'complete'
|
|
55
55
|
Requires-Dist: pystac[urllib3]>=1.8.2; extra == 'complete'
|
|
56
56
|
Requires-Dist: requests; extra == 'complete'
|
|
57
|
-
Requires-Dist: rtree; extra == 'complete'
|
|
58
57
|
Requires-Dist: s3fs!=2023.9.0; extra == 'complete'
|
|
59
58
|
Requires-Dist: tilebench; extra == 'complete'
|
|
60
59
|
Requires-Dist: werkzeug>=0.15; extra == 'complete'
|
|
@@ -76,8 +75,6 @@ Provides-Extra: serve
|
|
|
76
75
|
Requires-Dist: flask-rangerequest; extra == 'serve'
|
|
77
76
|
Requires-Dist: flask<4.0.0; extra == 'serve'
|
|
78
77
|
Requires-Dist: werkzeug>=0.15; extra == 'serve'
|
|
79
|
-
Provides-Extra: spatial-index
|
|
80
|
-
Requires-Dist: rtree; extra == 'spatial-index'
|
|
81
78
|
Provides-Extra: test
|
|
82
79
|
Requires-Dist: coverage>=5.1; extra == 'test'
|
|
83
80
|
Requires-Dist: coveralls>=2.2.0; extra == 'test'
|
|
@@ -93,6 +90,7 @@ Requires-Dist: pytest-mock; extra == 'test'
|
|
|
93
90
|
Requires-Dist: pytest-split; extra == 'test'
|
|
94
91
|
Requires-Dist: pytest<8,>=5.4.1; extra == 'test'
|
|
95
92
|
Requires-Dist: rio-cogeo>=5.0.0; extra == 'test'
|
|
93
|
+
Requires-Dist: rtree; extra == 'test'
|
|
96
94
|
Provides-Extra: vrt
|
|
97
95
|
Requires-Dist: lxml; extra == 'vrt'
|
|
98
96
|
Description-Content-Type: text/x-rst
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import click
|
|
2
|
+
import tqdm
|
|
3
|
+
|
|
4
|
+
from mapchete import commands
|
|
5
|
+
from mapchete.cli import options
|
|
6
|
+
from mapchete.cli.progress_bar import PBar
|
|
7
|
+
from mapchete.commands.rm import existing_paths, gen_existing_paths
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@click.command(help="Remove tiles from TileDirectory.")
|
|
11
|
+
@options.arg_tiledir
|
|
12
|
+
@options.opt_zoom
|
|
13
|
+
@options.opt_area
|
|
14
|
+
@options.opt_area_crs
|
|
15
|
+
@options.opt_bounds
|
|
16
|
+
@options.opt_bounds_crs
|
|
17
|
+
@options.opt_workers
|
|
18
|
+
@options.opt_verbose
|
|
19
|
+
@options.opt_no_pbar
|
|
20
|
+
@options.opt_debug
|
|
21
|
+
@options.opt_logfile
|
|
22
|
+
@options.opt_force
|
|
23
|
+
def rm(
|
|
24
|
+
*args,
|
|
25
|
+
force=False,
|
|
26
|
+
debug=False,
|
|
27
|
+
no_pbar=False,
|
|
28
|
+
verbose=False,
|
|
29
|
+
logfile=None,
|
|
30
|
+
**kwargs,
|
|
31
|
+
):
|
|
32
|
+
"""Remove tiles from TileDirectory."""
|
|
33
|
+
if force:
|
|
34
|
+
with PBar(
|
|
35
|
+
desc="tiles",
|
|
36
|
+
disable=debug or no_pbar,
|
|
37
|
+
print_messages=verbose,
|
|
38
|
+
) as pbar:
|
|
39
|
+
commands.rm(
|
|
40
|
+
paths=gen_existing_paths(*args, **kwargs), observers=[pbar], **kwargs
|
|
41
|
+
)
|
|
42
|
+
else:
|
|
43
|
+
tiles_to_delete = existing_paths(
|
|
44
|
+
*args,
|
|
45
|
+
**kwargs,
|
|
46
|
+
)
|
|
47
|
+
if len(tiles_to_delete):
|
|
48
|
+
if click.confirm(
|
|
49
|
+
f"Do you want to delete {len(tiles_to_delete)} tiles?", abort=True
|
|
50
|
+
):
|
|
51
|
+
with PBar(
|
|
52
|
+
total=len(tiles_to_delete),
|
|
53
|
+
desc="tiles",
|
|
54
|
+
disable=debug or no_pbar,
|
|
55
|
+
print_messages=verbose,
|
|
56
|
+
) as pbar:
|
|
57
|
+
commands.rm(paths=tiles_to_delete, observers=[pbar], **kwargs)
|
|
58
|
+
else: # pragma: no cover
|
|
59
|
+
tqdm.tqdm.write("No tiles found to delete.")
|
|
@@ -38,7 +38,9 @@ class PBar(ObserverProtocol):
|
|
|
38
38
|
if self._pbar.total is None or self._pbar.total != progress.total:
|
|
39
39
|
self._pbar.reset(progress.total) # type: ignore
|
|
40
40
|
|
|
41
|
-
self._pbar.update(progress.current - self._pbar.n)
|
|
41
|
+
# self._pbar.update(progress.current - self._pbar.n)
|
|
42
|
+
self._pbar.n = progress.current
|
|
43
|
+
self._pbar.update(n=0)
|
|
42
44
|
|
|
43
45
|
if self.print_messages:
|
|
44
46
|
if task_info:
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
"""Remove tiles from Tile Directory."""
|
|
2
2
|
|
|
3
|
+
from itertools import chain, islice
|
|
3
4
|
import logging
|
|
4
|
-
from typing import List, Optional, Union, Dict, Any
|
|
5
|
+
from typing import List, Optional, Union, Dict, Any, Generator, Iterable
|
|
5
6
|
|
|
6
7
|
from rasterio.crs import CRS
|
|
7
8
|
from shapely.geometry.base import BaseGeometry
|
|
@@ -11,14 +12,13 @@ from mapchete.commands.observer import ObserverProtocol, Observers
|
|
|
11
12
|
from mapchete.io import tiles_exist
|
|
12
13
|
from mapchete.path import MPath
|
|
13
14
|
from mapchete.types import MPathLike, Progress, BoundsLike
|
|
14
|
-
from mapchete.tile import BufferedTile
|
|
15
15
|
|
|
16
16
|
logger = logging.getLogger(__name__)
|
|
17
17
|
|
|
18
18
|
|
|
19
19
|
def rm(
|
|
20
20
|
tiledir: Optional[MPathLike] = None,
|
|
21
|
-
paths: Optional[
|
|
21
|
+
paths: Optional[Iterable[MPath]] = None,
|
|
22
22
|
zoom: Optional[Union[int, List[int]]] = None,
|
|
23
23
|
area: Union[BaseGeometry, str, dict] = None,
|
|
24
24
|
area_crs: Union[CRS, str] = None,
|
|
@@ -27,6 +27,7 @@ def rm(
|
|
|
27
27
|
workers: Optional[int] = None,
|
|
28
28
|
fs_opts: Optional[Dict[str, Any]] = None,
|
|
29
29
|
observers: Optional[List[ObserverProtocol]] = None,
|
|
30
|
+
s3_delete_chunksize: int = 1000,
|
|
30
31
|
):
|
|
31
32
|
"""
|
|
32
33
|
Remove tiles from TileDirectory.
|
|
@@ -50,13 +51,13 @@ def rm(
|
|
|
50
51
|
Configuration options for fsspec filesystem.
|
|
51
52
|
"""
|
|
52
53
|
all_observers = Observers(observers)
|
|
53
|
-
|
|
54
|
-
|
|
54
|
+
if paths is not None:
|
|
55
|
+
pass
|
|
56
|
+
elif tiledir:
|
|
55
57
|
if zoom is None: # pragma: no cover
|
|
56
58
|
raise ValueError("zoom level(s) required")
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
tiledir=tiledir,
|
|
59
|
+
paths = gen_existing_paths(
|
|
60
|
+
tiledir=MPath.from_inp(tiledir, storage_options=fs_opts),
|
|
60
61
|
zoom=zoom,
|
|
61
62
|
area=area,
|
|
62
63
|
area_crs=area_crs,
|
|
@@ -64,44 +65,65 @@ def rm(
|
|
|
64
65
|
bounds_crs=bounds_crs,
|
|
65
66
|
workers=workers,
|
|
66
67
|
)
|
|
67
|
-
elif isinstance(paths, list):
|
|
68
|
-
pass
|
|
69
68
|
else: # pragma: no cover
|
|
70
|
-
raise ValueError(
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
69
|
+
raise ValueError("either tiledir or paths have to be provided")
|
|
70
|
+
|
|
71
|
+
try:
|
|
72
|
+
# this only works for lists
|
|
73
|
+
total = len(paths)
|
|
74
|
+
all_observers.notify(progress=Progress(total=total))
|
|
75
|
+
logger.debug("got %s path(s)", len(paths))
|
|
76
|
+
except TypeError:
|
|
77
|
+
# we don't know the length of a generator in advance
|
|
78
|
+
total = None
|
|
79
|
+
|
|
80
|
+
# create an iterator to have a common interface for lists and generators
|
|
81
|
+
paths_iter = iter(paths)
|
|
82
|
+
# determine the filesystem by getting the first path
|
|
83
|
+
try:
|
|
84
|
+
first_path = next(paths_iter)
|
|
85
|
+
except StopIteration:
|
|
86
|
+
# this happends if the list or generator does not have any item
|
|
75
87
|
logger.debug("no paths to delete")
|
|
76
88
|
return
|
|
77
89
|
|
|
78
|
-
|
|
79
|
-
|
|
90
|
+
fs = first_path.fs
|
|
91
|
+
# append first_path to other paths again
|
|
92
|
+
all_paths = chain((first_path,), paths_iter)
|
|
80
93
|
|
|
81
94
|
# s3fs enables multiple paths as input, so let's use this:
|
|
82
|
-
if "s3" in
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
msg = f"deleted {path}"
|
|
86
|
-
logger.debug(msg)
|
|
95
|
+
if "s3" in fs.protocol:
|
|
96
|
+
tiles_counter = 0
|
|
97
|
+
while chunk := tuple(islice(all_paths, s3_delete_chunksize)):
|
|
87
98
|
all_observers.notify(
|
|
88
|
-
progress=Progress(
|
|
99
|
+
progress=Progress(
|
|
100
|
+
current=tiles_counter, total=total or tiles_counter + len(chunk)
|
|
101
|
+
)
|
|
89
102
|
)
|
|
103
|
+
# this actually deletes the files
|
|
104
|
+
fs.rm(chunk)
|
|
105
|
+
for path in chunk:
|
|
106
|
+
tiles_counter += 1
|
|
107
|
+
msg = f"deleted {path}"
|
|
108
|
+
logger.debug(msg)
|
|
109
|
+
all_observers.notify(message=msg)
|
|
110
|
+
all_observers.notify(progress=Progress(current=tiles_counter))
|
|
90
111
|
|
|
91
112
|
# otherwise, just iterate through the paths
|
|
92
113
|
else:
|
|
93
|
-
for
|
|
114
|
+
for tiles_counter, path in enumerate(all_paths, 1):
|
|
94
115
|
path.rm()
|
|
95
116
|
msg = f"deleted {path}"
|
|
96
117
|
logger.debug(msg)
|
|
97
118
|
all_observers.notify(
|
|
98
|
-
progress=Progress(current=
|
|
119
|
+
progress=Progress(current=tiles_counter, total=total or tiles_counter),
|
|
120
|
+
message=msg,
|
|
99
121
|
)
|
|
100
122
|
|
|
101
|
-
all_observers.notify(message=f"{
|
|
123
|
+
all_observers.notify(message=f"{tiles_counter} tiles deleted")
|
|
102
124
|
|
|
103
125
|
|
|
104
|
-
def
|
|
126
|
+
def gen_existing_paths(
|
|
105
127
|
tiledir: MPathLike,
|
|
106
128
|
zoom: Optional[Union[int, List[int]]] = None,
|
|
107
129
|
area: Union[BaseGeometry, str, dict] = None,
|
|
@@ -109,7 +131,7 @@ def existing_paths(
|
|
|
109
131
|
bounds: Optional[BoundsLike] = None,
|
|
110
132
|
bounds_crs: Union[CRS, str] = None,
|
|
111
133
|
workers: Optional[int] = None,
|
|
112
|
-
) ->
|
|
134
|
+
) -> Generator[MPath, None, None]:
|
|
113
135
|
with mapchete.open(
|
|
114
136
|
tiledir,
|
|
115
137
|
zoom=zoom,
|
|
@@ -120,26 +142,39 @@ def existing_paths(
|
|
|
120
142
|
mode="readonly",
|
|
121
143
|
) as mp:
|
|
122
144
|
tp = mp.config.output_pyramid
|
|
123
|
-
tiles: Dict[Union[int, List[int], None], List[BufferedTile]] = {}
|
|
124
145
|
for zoom in mp.config.init_zoom_levels:
|
|
125
|
-
tiles[zoom] = []
|
|
126
146
|
# check which source tiles exist
|
|
127
147
|
logger.debug("looking for existing source tiles in zoom %s...", zoom)
|
|
128
148
|
for tile, exists in tiles_exist(
|
|
129
149
|
config=mp.config,
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
for tile_ in tp.tiles_from_geom(mp.config.area_at_zoom(zoom), zoom)
|
|
133
|
-
# this is required to omit tiles touching the config area
|
|
134
|
-
if mp.config.area_at_zoom(zoom).intersection(tile_.bbox).area
|
|
150
|
+
output_tiles_batches=tp.tiles_from_geom_batches(
|
|
151
|
+
mp.config.area_at_zoom(zoom), zoom
|
|
135
152
|
),
|
|
136
153
|
workers=workers,
|
|
137
154
|
):
|
|
138
155
|
if exists:
|
|
139
|
-
|
|
156
|
+
logger.debug("yield tile %s", tile)
|
|
157
|
+
yield mp.config.output_reader.get_path(tile)
|
|
158
|
+
|
|
140
159
|
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
160
|
+
def existing_paths(
|
|
161
|
+
tiledir: MPathLike,
|
|
162
|
+
zoom: Optional[Union[int, List[int]]] = None,
|
|
163
|
+
area: Union[BaseGeometry, str, dict] = None,
|
|
164
|
+
area_crs: Union[CRS, str] = None,
|
|
165
|
+
bounds: Optional[BoundsLike] = None,
|
|
166
|
+
bounds_crs: Union[CRS, str] = None,
|
|
167
|
+
workers: Optional[int] = None,
|
|
168
|
+
) -> List[MPath]:
|
|
169
|
+
return [
|
|
170
|
+
tile
|
|
171
|
+
for tile in gen_existing_paths(
|
|
172
|
+
tiledir=tiledir,
|
|
173
|
+
zoom=zoom,
|
|
174
|
+
area=area,
|
|
175
|
+
area_crs=area_crs,
|
|
176
|
+
bounds=bounds,
|
|
177
|
+
bounds_crs=bounds_crs,
|
|
178
|
+
workers=workers,
|
|
179
|
+
)
|
|
180
|
+
]
|
|
@@ -6,6 +6,7 @@ import warnings
|
|
|
6
6
|
from collections import OrderedDict
|
|
7
7
|
from copy import deepcopy
|
|
8
8
|
from functools import cached_property
|
|
9
|
+
import textwrap
|
|
9
10
|
from typing import Any, Iterator, Optional, Tuple, Union
|
|
10
11
|
|
|
11
12
|
import oyaml as yaml
|
|
@@ -123,7 +124,7 @@ class MapcheteConfig(object):
|
|
|
123
124
|
"""Initialize configuration."""
|
|
124
125
|
# get dictionary representation of input_config and
|
|
125
126
|
# (1) map deprecated params to new structure
|
|
126
|
-
logger.debug(
|
|
127
|
+
logger.debug("parsing %s", input_config)
|
|
127
128
|
try:
|
|
128
129
|
self.parsed_config = parse_config(input_config, strict=stric_parsing)
|
|
129
130
|
self.parsed_config.model_dump()
|
|
@@ -240,9 +241,12 @@ class MapcheteConfig(object):
|
|
|
240
241
|
area_crs=area_crs,
|
|
241
242
|
bounds_crs=bounds_crs,
|
|
242
243
|
)
|
|
243
|
-
logger.info(
|
|
244
|
+
logger.info(
|
|
245
|
+
"process area: %s",
|
|
246
|
+
textwrap.shorten(self.area.wkt, width=200, placeholder="..."),
|
|
247
|
+
)
|
|
244
248
|
self.bounds = Bounds(*self.area.bounds)
|
|
245
|
-
logger.debug(
|
|
249
|
+
logger.debug("process bounds: %s", self.bounds)
|
|
246
250
|
self.init_area = self._get_process_area(
|
|
247
251
|
area=self._init_area,
|
|
248
252
|
bounds=self._init_bounds,
|
|
@@ -251,9 +255,12 @@ class MapcheteConfig(object):
|
|
|
251
255
|
area_crs=area_crs,
|
|
252
256
|
bounds_crs=bounds_crs,
|
|
253
257
|
)
|
|
254
|
-
logger.info(
|
|
258
|
+
logger.info(
|
|
259
|
+
"init area: %s",
|
|
260
|
+
textwrap.shorten(self.init_area.wkt, width=200, placeholder="..."),
|
|
261
|
+
)
|
|
255
262
|
self.init_bounds = Bounds(*self.init_area.bounds)
|
|
256
|
-
logger.debug(
|
|
263
|
+
logger.debug("init bounds: %s", self.init_bounds)
|
|
257
264
|
|
|
258
265
|
# (7) the delimiters are used by some input drivers
|
|
259
266
|
self._delimiters
|
|
@@ -211,7 +211,7 @@ class InputTile(base.InputTile, VectorInput):
|
|
|
211
211
|
raise NotImplementedError()
|
|
212
212
|
return self._from_cache(validity_check=validity_check)
|
|
213
213
|
|
|
214
|
-
def is_empty(self, validity_check=True): # pragma: no cover
|
|
214
|
+
def is_empty(self, validity_check=True, **_): # pragma: no cover
|
|
215
215
|
"""
|
|
216
216
|
Check if there is data within this tile.
|
|
217
217
|
|
|
@@ -714,7 +714,7 @@ class InputTile(base.InputTile, RasterInput):
|
|
|
714
714
|
else ma.concatenate([ma.expand_dims(arr[i - 1], 0) for i in band_indexes])
|
|
715
715
|
)
|
|
716
716
|
|
|
717
|
-
def is_empty(self,
|
|
717
|
+
def is_empty(self, **_):
|
|
718
718
|
"""
|
|
719
719
|
Check if there is data within this tile.
|
|
720
720
|
|
|
@@ -269,7 +269,7 @@ class ReferencedRasterInput(RasterInput):
|
|
|
269
269
|
"""Either read full array or resampled to grid."""
|
|
270
270
|
return self.referenced_raster.read(indexes=indexes, resampling=resampling)
|
|
271
271
|
|
|
272
|
-
def is_empty(self) -> bool:
|
|
272
|
+
def is_empty(self, **_) -> bool:
|
|
273
273
|
"""Checks if input is empty here."""
|
|
274
274
|
return self.referenced_raster.masked_array().mask.all()
|
|
275
275
|
|