mapchete 2026.2.2__tar.gz → 2026.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {mapchete-2026.2.2 → mapchete-2026.4.0}/PKG-INFO +2 -4
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/__init__.py +1 -1
- mapchete-2026.4.0/mapchete/cli/mapchete/rm.py +59 -0
- mapchete-2026.4.0/mapchete/cli/mpath/rm.py +115 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/cli/mpath/sync.py +11 -2
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/cli/progress_bar.py +3 -1
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/commands/rm.py +76 -41
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/config/base.py +12 -5
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/executor/concurrent_futures.py +9 -5
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/formats/default/_fiona_base.py +1 -1
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/formats/default/gtiff.py +1 -1
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/formats/default/raster_file.py +1 -1
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/formats/default/tile_directory.py +1 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/formats/default/vector_file.py +1 -1
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/formats/protocols.py +1 -1
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/geometry/clip.py +31 -1
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/geometry/shape.py +9 -5
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/io/raster/referenced_raster.py +1 -1
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/io/vector/indexed_features.py +122 -56
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/io/vector/read.py +5 -5
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/path.py +26 -13
- {mapchete-2026.2.2 → mapchete-2026.4.0}/pyproject.toml +1 -4
- mapchete-2026.2.2/mapchete/cli/mapchete/rm.py +0 -49
- mapchete-2026.2.2/mapchete/cli/mpath/rm.py +0 -19
- {mapchete-2026.2.2 → mapchete-2026.4.0}/.gitignore +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/LICENSE +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/README.rst +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/_deprecated.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/bounds.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/cli/__init__.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/cli/mapchete/__init__.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/cli/mapchete/convert.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/cli/mapchete/cp.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/cli/mapchete/create.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/cli/mapchete/execute.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/cli/mapchete/formats.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/cli/mapchete/index.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/cli/mapchete/processes.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/cli/mapchete/serve.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/cli/mapchete/show_env.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/cli/mapchete/stac.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/cli/mpath/__init__.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/cli/mpath/cp.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/cli/mpath/exists.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/cli/mpath/ls.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/cli/mpath/read_json.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/cli/mpath/read_text.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/cli/mpath/read_yaml.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/cli/options.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/commands/__init__.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/commands/convert.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/commands/cp.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/commands/execute.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/commands/index.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/commands/observer.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/commands/parser.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/config/__init__.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/config/models.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/config/parse.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/config/process_func.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/enums.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/errors.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/executor/__init__.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/executor/base.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/executor/dask.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/executor/future.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/executor/sequential.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/executor/types.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/formats/__init__.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/formats/base.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/formats/default/__init__.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/formats/default/flatgeobuf.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/formats/default/geojson.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/formats/default/mapchete_input.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/formats/default/png.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/formats/default/png_hillshade.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/formats/drivers.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/formats/loaders.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/formats/tools.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/geometry/__init__.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/geometry/filter.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/geometry/footprints.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/geometry/latlon.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/geometry/repair.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/geometry/reproject.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/geometry/segmentize.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/geometry/transform.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/geometry/types.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/grid.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/index.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/io/__init__.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/io/_json.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/io/_misc.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/io/_path.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/io/profiles.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/io/raster/__init__.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/io/raster/array.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/io/raster/convert.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/io/raster/mosaic.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/io/raster/open.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/io/raster/read.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/io/raster/write.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/io/vector/__init__.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/io/vector/convert.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/io/vector/open.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/io/vector/types.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/io/vector/write.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/log.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/pretty.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/process_func_special_types.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/processes/__init__.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/processes/clip.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/processes/contours.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/processes/convert.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/processes/examples/__init__.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/processes/examples/example_process.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/processes/hillshade.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/processing/__init__.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/processing/base.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/processing/execute.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/processing/mp.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/processing/profilers/__init__.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/processing/profilers/memory.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/processing/profilers/requests.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/processing/profilers/time.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/processing/tasks.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/processing/types.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/protocols.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/registered.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/settings.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/stac/__init__.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/stac/models.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/stac/tiled_assets.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/static/__init__.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/static/index.html +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/static/mapchete_template.mapchete +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/static/process_template.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/testing.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/tile.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/timer.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/types.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/validate.py +0 -0
- {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/zoom_levels.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: mapchete
|
|
3
|
-
Version: 2026.
|
|
3
|
+
Version: 2026.4.0
|
|
4
4
|
Summary: Tile-based geodata processing using rasterio & Fiona
|
|
5
5
|
Project-URL: Homepage, https://github.com/ungarj/mapchete
|
|
6
6
|
Author-email: Joachim Ungar <joachim.ungar@gmail.com>
|
|
@@ -54,7 +54,6 @@ Requires-Dist: matplotlib<3.11; extra == 'complete'
|
|
|
54
54
|
Requires-Dist: memray; extra == 'complete'
|
|
55
55
|
Requires-Dist: pystac[urllib3]>=1.8.2; extra == 'complete'
|
|
56
56
|
Requires-Dist: requests; extra == 'complete'
|
|
57
|
-
Requires-Dist: rtree; extra == 'complete'
|
|
58
57
|
Requires-Dist: s3fs!=2023.9.0; extra == 'complete'
|
|
59
58
|
Requires-Dist: tilebench; extra == 'complete'
|
|
60
59
|
Requires-Dist: werkzeug>=0.15; extra == 'complete'
|
|
@@ -76,8 +75,6 @@ Provides-Extra: serve
|
|
|
76
75
|
Requires-Dist: flask-rangerequest; extra == 'serve'
|
|
77
76
|
Requires-Dist: flask<4.0.0; extra == 'serve'
|
|
78
77
|
Requires-Dist: werkzeug>=0.15; extra == 'serve'
|
|
79
|
-
Provides-Extra: spatial-index
|
|
80
|
-
Requires-Dist: rtree; extra == 'spatial-index'
|
|
81
78
|
Provides-Extra: test
|
|
82
79
|
Requires-Dist: coverage>=5.1; extra == 'test'
|
|
83
80
|
Requires-Dist: coveralls>=2.2.0; extra == 'test'
|
|
@@ -93,6 +90,7 @@ Requires-Dist: pytest-mock; extra == 'test'
|
|
|
93
90
|
Requires-Dist: pytest-split; extra == 'test'
|
|
94
91
|
Requires-Dist: pytest<8,>=5.4.1; extra == 'test'
|
|
95
92
|
Requires-Dist: rio-cogeo>=5.0.0; extra == 'test'
|
|
93
|
+
Requires-Dist: rtree; extra == 'test'
|
|
96
94
|
Provides-Extra: vrt
|
|
97
95
|
Requires-Dist: lxml; extra == 'vrt'
|
|
98
96
|
Description-Content-Type: text/x-rst
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import click
|
|
2
|
+
import tqdm
|
|
3
|
+
|
|
4
|
+
from mapchete import commands
|
|
5
|
+
from mapchete.cli import options
|
|
6
|
+
from mapchete.cli.progress_bar import PBar
|
|
7
|
+
from mapchete.commands.rm import existing_paths, gen_existing_paths
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@click.command(help="Remove tiles from TileDirectory.")
|
|
11
|
+
@options.arg_tiledir
|
|
12
|
+
@options.opt_zoom
|
|
13
|
+
@options.opt_area
|
|
14
|
+
@options.opt_area_crs
|
|
15
|
+
@options.opt_bounds
|
|
16
|
+
@options.opt_bounds_crs
|
|
17
|
+
@options.opt_workers
|
|
18
|
+
@options.opt_verbose
|
|
19
|
+
@options.opt_no_pbar
|
|
20
|
+
@options.opt_debug
|
|
21
|
+
@options.opt_logfile
|
|
22
|
+
@options.opt_force
|
|
23
|
+
def rm(
|
|
24
|
+
*args,
|
|
25
|
+
force=False,
|
|
26
|
+
debug=False,
|
|
27
|
+
no_pbar=False,
|
|
28
|
+
verbose=False,
|
|
29
|
+
logfile=None,
|
|
30
|
+
**kwargs,
|
|
31
|
+
):
|
|
32
|
+
"""Remove tiles from TileDirectory."""
|
|
33
|
+
if force:
|
|
34
|
+
with PBar(
|
|
35
|
+
desc="tiles",
|
|
36
|
+
disable=debug or no_pbar,
|
|
37
|
+
print_messages=verbose,
|
|
38
|
+
) as pbar:
|
|
39
|
+
commands.rm(
|
|
40
|
+
paths=gen_existing_paths(*args, **kwargs), observers=[pbar], **kwargs
|
|
41
|
+
)
|
|
42
|
+
else:
|
|
43
|
+
tiles_to_delete = existing_paths(
|
|
44
|
+
*args,
|
|
45
|
+
**kwargs,
|
|
46
|
+
)
|
|
47
|
+
if len(tiles_to_delete):
|
|
48
|
+
if click.confirm(
|
|
49
|
+
f"Do you want to delete {len(tiles_to_delete)} tiles?", abort=True
|
|
50
|
+
):
|
|
51
|
+
with PBar(
|
|
52
|
+
total=len(tiles_to_delete),
|
|
53
|
+
desc="tiles",
|
|
54
|
+
disable=debug or no_pbar,
|
|
55
|
+
print_messages=verbose,
|
|
56
|
+
) as pbar:
|
|
57
|
+
commands.rm(paths=tiles_to_delete, observers=[pbar], **kwargs)
|
|
58
|
+
else: # pragma: no cover
|
|
59
|
+
tqdm.tqdm.write("No tiles found to delete.")
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
from typing import Optional
|
|
3
|
+
|
|
4
|
+
import click
|
|
5
|
+
import click_spinner
|
|
6
|
+
import tqdm
|
|
7
|
+
|
|
8
|
+
from mapchete.cli import options
|
|
9
|
+
from mapchete.enums import Concurrency
|
|
10
|
+
from mapchete.executor import get_executor
|
|
11
|
+
from mapchete.path import MPath
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
logger = logging.getLogger(__name__)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@click.command(help="Remove path.")
|
|
18
|
+
@options.arg_path
|
|
19
|
+
@options.opt_src_fs_opts
|
|
20
|
+
@options.opt_recursive
|
|
21
|
+
@options.opt_force
|
|
22
|
+
@options.opt_verbose
|
|
23
|
+
@options.opt_workers
|
|
24
|
+
@click.option(
|
|
25
|
+
"--count",
|
|
26
|
+
is_flag=True,
|
|
27
|
+
help="Count all files from source path. WARNING: this will trigger more requests on S3.",
|
|
28
|
+
)
|
|
29
|
+
def rm(
|
|
30
|
+
path: MPath,
|
|
31
|
+
recursive: bool = False,
|
|
32
|
+
force: bool = False,
|
|
33
|
+
verbose: bool = True,
|
|
34
|
+
count: bool = False,
|
|
35
|
+
debug: bool = False,
|
|
36
|
+
workers: Optional[int] = None,
|
|
37
|
+
**_,
|
|
38
|
+
):
|
|
39
|
+
if force or click.confirm(f"do you really want to permanently delete {str(path)}?"):
|
|
40
|
+
try:
|
|
41
|
+
if path.is_directory():
|
|
42
|
+
if recursive is False: # pragma: no cover
|
|
43
|
+
raise click.UsageError(
|
|
44
|
+
"--recursive flag has to be active if path is a directory"
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
with get_executor(
|
|
48
|
+
concurrency=Concurrency.none
|
|
49
|
+
if workers == 1
|
|
50
|
+
else Concurrency.threads,
|
|
51
|
+
max_workers=workers,
|
|
52
|
+
) as executor:
|
|
53
|
+
if count:
|
|
54
|
+
click.echo("counting pages ...")
|
|
55
|
+
with click_spinner.Spinner(disable=debug):
|
|
56
|
+
pages = list(path.paginate())
|
|
57
|
+
click.echo(f"found {len(pages)} pages")
|
|
58
|
+
else:
|
|
59
|
+
pages = path.paginate()
|
|
60
|
+
deleted = 0
|
|
61
|
+
with tqdm.tqdm(
|
|
62
|
+
total=len(pages) if isinstance(pages, list) else None,
|
|
63
|
+
desc="pages",
|
|
64
|
+
) as pbar:
|
|
65
|
+
for page in pages:
|
|
66
|
+
# when using a generator, manually update total length
|
|
67
|
+
if not isinstance(pages, list):
|
|
68
|
+
pbar.total = (pbar.total or 0) + 1
|
|
69
|
+
pbar.refresh()
|
|
70
|
+
|
|
71
|
+
if verbose: # pragma: no cover
|
|
72
|
+
tqdm.tqdm.write(f"found {len(page)} files")
|
|
73
|
+
logger.debug("found %s files", len(page))
|
|
74
|
+
|
|
75
|
+
for future in tqdm.tqdm(
|
|
76
|
+
executor.as_completed(
|
|
77
|
+
rm_file,
|
|
78
|
+
page,
|
|
79
|
+
max_submitted_tasks=workers * 10,
|
|
80
|
+
),
|
|
81
|
+
disable=debug,
|
|
82
|
+
total=len(page),
|
|
83
|
+
desc="files",
|
|
84
|
+
leave=False,
|
|
85
|
+
):
|
|
86
|
+
future.raise_if_failed()
|
|
87
|
+
if verbose:
|
|
88
|
+
msg = future.result()
|
|
89
|
+
tqdm.tqdm.write(msg)
|
|
90
|
+
|
|
91
|
+
deleted += len(page)
|
|
92
|
+
pbar.update()
|
|
93
|
+
|
|
94
|
+
# finally delete folder after all its contents have been deleted
|
|
95
|
+
path.rm(recursive=True)
|
|
96
|
+
click.echo(f"{deleted} files deleted")
|
|
97
|
+
|
|
98
|
+
else:
|
|
99
|
+
msg = rm_file(path)
|
|
100
|
+
if verbose: # pragma: no cover
|
|
101
|
+
tqdm.tqdm.write(msg)
|
|
102
|
+
|
|
103
|
+
except Exception as exc: # pragma: no cover
|
|
104
|
+
raise click.ClickException(str(exc))
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def rm_file(file: MPath) -> str:
|
|
108
|
+
try:
|
|
109
|
+
file.rm()
|
|
110
|
+
msg = f"[OK] {str(file)}: deleted"
|
|
111
|
+
logger.debug(msg)
|
|
112
|
+
return msg
|
|
113
|
+
except Exception as exc: # pragma: no cover
|
|
114
|
+
logger.exception(exc)
|
|
115
|
+
raise
|
|
@@ -42,6 +42,7 @@ logger = logging.getLogger(__name__)
|
|
|
42
42
|
@options.opt_workers
|
|
43
43
|
@options.opt_debug
|
|
44
44
|
@options.opt_verbose
|
|
45
|
+
@click.option("--dry-run", is_flag=True, help="Don't copy anything.")
|
|
45
46
|
def sync(
|
|
46
47
|
path: MPath,
|
|
47
48
|
out_path: MPath,
|
|
@@ -51,6 +52,7 @@ def sync(
|
|
|
51
52
|
workers: int = 1,
|
|
52
53
|
debug: bool = False,
|
|
53
54
|
verbose: bool = False,
|
|
55
|
+
dry_run: bool = False,
|
|
54
56
|
**_,
|
|
55
57
|
):
|
|
56
58
|
try:
|
|
@@ -83,7 +85,7 @@ def sync(
|
|
|
83
85
|
path, out_path, compare_checksums=compare_checksums
|
|
84
86
|
),
|
|
85
87
|
fargs=None,
|
|
86
|
-
fkwargs=dict(chunksize=chunksize, debug=debug),
|
|
88
|
+
fkwargs=dict(chunksize=chunksize, debug=debug, dry_run=dry_run),
|
|
87
89
|
item_skip_bool=True,
|
|
88
90
|
max_submitted_tasks=workers * 10,
|
|
89
91
|
),
|
|
@@ -144,8 +146,15 @@ def check_files(
|
|
|
144
146
|
|
|
145
147
|
|
|
146
148
|
def sync_file(
|
|
147
|
-
paths: Tuple[MPath, MPath],
|
|
149
|
+
paths: Tuple[MPath, MPath],
|
|
150
|
+
chunksize: int = 1024 * 1024,
|
|
151
|
+
debug: bool = False,
|
|
152
|
+
dry_run: bool = False,
|
|
148
153
|
) -> Tuple[Tuple[MPath, MPath], str]:
|
|
154
|
+
|
|
155
|
+
if dry_run: # pragma: no cover
|
|
156
|
+
return paths, "0s [DRY-RUN]"
|
|
157
|
+
|
|
149
158
|
src_file, dst_file = paths
|
|
150
159
|
with PBar(
|
|
151
160
|
total=100,
|
|
@@ -38,7 +38,9 @@ class PBar(ObserverProtocol):
|
|
|
38
38
|
if self._pbar.total is None or self._pbar.total != progress.total:
|
|
39
39
|
self._pbar.reset(progress.total) # type: ignore
|
|
40
40
|
|
|
41
|
-
self._pbar.update(progress.current - self._pbar.n)
|
|
41
|
+
# self._pbar.update(progress.current - self._pbar.n)
|
|
42
|
+
self._pbar.n = progress.current
|
|
43
|
+
self._pbar.update(n=0)
|
|
42
44
|
|
|
43
45
|
if self.print_messages:
|
|
44
46
|
if task_info:
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
"""Remove tiles from Tile Directory."""
|
|
2
2
|
|
|
3
|
+
from itertools import chain, islice
|
|
3
4
|
import logging
|
|
4
|
-
from typing import List, Optional, Union, Dict, Any
|
|
5
|
+
from typing import List, Optional, Union, Dict, Any, Generator, Iterable
|
|
5
6
|
|
|
6
7
|
from rasterio.crs import CRS
|
|
7
8
|
from shapely.geometry.base import BaseGeometry
|
|
@@ -11,14 +12,13 @@ from mapchete.commands.observer import ObserverProtocol, Observers
|
|
|
11
12
|
from mapchete.io import tiles_exist
|
|
12
13
|
from mapchete.path import MPath
|
|
13
14
|
from mapchete.types import MPathLike, Progress, BoundsLike
|
|
14
|
-
from mapchete.tile import BufferedTile
|
|
15
15
|
|
|
16
16
|
logger = logging.getLogger(__name__)
|
|
17
17
|
|
|
18
18
|
|
|
19
19
|
def rm(
|
|
20
20
|
tiledir: Optional[MPathLike] = None,
|
|
21
|
-
paths: Optional[
|
|
21
|
+
paths: Optional[Iterable[MPath]] = None,
|
|
22
22
|
zoom: Optional[Union[int, List[int]]] = None,
|
|
23
23
|
area: Union[BaseGeometry, str, dict] = None,
|
|
24
24
|
area_crs: Union[CRS, str] = None,
|
|
@@ -27,6 +27,7 @@ def rm(
|
|
|
27
27
|
workers: Optional[int] = None,
|
|
28
28
|
fs_opts: Optional[Dict[str, Any]] = None,
|
|
29
29
|
observers: Optional[List[ObserverProtocol]] = None,
|
|
30
|
+
s3_delete_chunksize: int = 1000,
|
|
30
31
|
):
|
|
31
32
|
"""
|
|
32
33
|
Remove tiles from TileDirectory.
|
|
@@ -50,13 +51,13 @@ def rm(
|
|
|
50
51
|
Configuration options for fsspec filesystem.
|
|
51
52
|
"""
|
|
52
53
|
all_observers = Observers(observers)
|
|
53
|
-
|
|
54
|
-
|
|
54
|
+
if paths is not None:
|
|
55
|
+
pass
|
|
56
|
+
elif tiledir:
|
|
55
57
|
if zoom is None: # pragma: no cover
|
|
56
58
|
raise ValueError("zoom level(s) required")
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
tiledir=tiledir,
|
|
59
|
+
paths = gen_existing_paths(
|
|
60
|
+
tiledir=MPath.from_inp(tiledir, storage_options=fs_opts),
|
|
60
61
|
zoom=zoom,
|
|
61
62
|
area=area,
|
|
62
63
|
area_crs=area_crs,
|
|
@@ -64,44 +65,65 @@ def rm(
|
|
|
64
65
|
bounds_crs=bounds_crs,
|
|
65
66
|
workers=workers,
|
|
66
67
|
)
|
|
67
|
-
elif isinstance(paths, list):
|
|
68
|
-
pass
|
|
69
68
|
else: # pragma: no cover
|
|
70
|
-
raise ValueError(
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
69
|
+
raise ValueError("either tiledir or paths have to be provided")
|
|
70
|
+
|
|
71
|
+
try:
|
|
72
|
+
# this only works for lists
|
|
73
|
+
total = len(paths)
|
|
74
|
+
all_observers.notify(progress=Progress(total=total))
|
|
75
|
+
logger.debug("got %s path(s)", len(paths))
|
|
76
|
+
except TypeError:
|
|
77
|
+
# we don't know the length of a generator in advance
|
|
78
|
+
total = None
|
|
79
|
+
|
|
80
|
+
# create an iterator to have a common interface for lists and generators
|
|
81
|
+
paths_iter = iter(paths)
|
|
82
|
+
# determine the filesystem by getting the first path
|
|
83
|
+
try:
|
|
84
|
+
first_path = next(paths_iter)
|
|
85
|
+
except StopIteration:
|
|
86
|
+
# this happends if the list or generator does not have any item
|
|
75
87
|
logger.debug("no paths to delete")
|
|
76
88
|
return
|
|
77
89
|
|
|
78
|
-
|
|
79
|
-
|
|
90
|
+
fs = first_path.fs
|
|
91
|
+
# append first_path to other paths again
|
|
92
|
+
all_paths = chain((first_path,), paths_iter)
|
|
80
93
|
|
|
81
94
|
# s3fs enables multiple paths as input, so let's use this:
|
|
82
|
-
if "s3" in
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
msg = f"deleted {path}"
|
|
86
|
-
logger.debug(msg)
|
|
95
|
+
if "s3" in fs.protocol:
|
|
96
|
+
tiles_counter = 0
|
|
97
|
+
while chunk := tuple(islice(all_paths, s3_delete_chunksize)):
|
|
87
98
|
all_observers.notify(
|
|
88
|
-
progress=Progress(
|
|
99
|
+
progress=Progress(
|
|
100
|
+
current=tiles_counter, total=total or tiles_counter + len(chunk)
|
|
101
|
+
)
|
|
89
102
|
)
|
|
103
|
+
# this actually deletes the files
|
|
104
|
+
fs.rm(chunk)
|
|
105
|
+
for path in chunk:
|
|
106
|
+
tiles_counter += 1
|
|
107
|
+
msg = f"deleted {path}"
|
|
108
|
+
logger.debug(msg)
|
|
109
|
+
all_observers.notify(message=msg)
|
|
110
|
+
all_observers.notify(progress=Progress(current=tiles_counter))
|
|
90
111
|
|
|
91
112
|
# otherwise, just iterate through the paths
|
|
92
113
|
else:
|
|
93
|
-
for
|
|
114
|
+
for tiles_counter, path in enumerate(all_paths, 1):
|
|
94
115
|
path.rm()
|
|
95
116
|
msg = f"deleted {path}"
|
|
96
117
|
logger.debug(msg)
|
|
97
118
|
all_observers.notify(
|
|
98
|
-
progress=Progress(current=
|
|
119
|
+
progress=Progress(current=tiles_counter, total=total or tiles_counter),
|
|
120
|
+
message=msg,
|
|
99
121
|
)
|
|
100
122
|
|
|
101
|
-
all_observers.notify(message=f"{
|
|
123
|
+
all_observers.notify(message=f"{tiles_counter} tiles deleted")
|
|
102
124
|
|
|
103
125
|
|
|
104
|
-
def
|
|
126
|
+
def gen_existing_paths(
|
|
105
127
|
tiledir: MPathLike,
|
|
106
128
|
zoom: Optional[Union[int, List[int]]] = None,
|
|
107
129
|
area: Union[BaseGeometry, str, dict] = None,
|
|
@@ -109,7 +131,7 @@ def existing_paths(
|
|
|
109
131
|
bounds: Optional[BoundsLike] = None,
|
|
110
132
|
bounds_crs: Union[CRS, str] = None,
|
|
111
133
|
workers: Optional[int] = None,
|
|
112
|
-
) ->
|
|
134
|
+
) -> Generator[MPath, None, None]:
|
|
113
135
|
with mapchete.open(
|
|
114
136
|
tiledir,
|
|
115
137
|
zoom=zoom,
|
|
@@ -120,26 +142,39 @@ def existing_paths(
|
|
|
120
142
|
mode="readonly",
|
|
121
143
|
) as mp:
|
|
122
144
|
tp = mp.config.output_pyramid
|
|
123
|
-
tiles: Dict[Union[int, List[int], None], List[BufferedTile]] = {}
|
|
124
145
|
for zoom in mp.config.init_zoom_levels:
|
|
125
|
-
tiles[zoom] = []
|
|
126
146
|
# check which source tiles exist
|
|
127
147
|
logger.debug("looking for existing source tiles in zoom %s...", zoom)
|
|
128
148
|
for tile, exists in tiles_exist(
|
|
129
149
|
config=mp.config,
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
for tile_ in tp.tiles_from_geom(mp.config.area_at_zoom(zoom), zoom)
|
|
133
|
-
# this is required to omit tiles touching the config area
|
|
134
|
-
if mp.config.area_at_zoom(zoom).intersection(tile_.bbox).area
|
|
150
|
+
output_tiles_batches=tp.tiles_from_geom_batches(
|
|
151
|
+
mp.config.area_at_zoom(zoom), zoom
|
|
135
152
|
),
|
|
136
153
|
workers=workers,
|
|
137
154
|
):
|
|
138
155
|
if exists:
|
|
139
|
-
|
|
156
|
+
logger.debug("yield tile %s", tile)
|
|
157
|
+
yield mp.config.output_reader.get_path(tile)
|
|
158
|
+
|
|
140
159
|
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
160
|
+
def existing_paths(
|
|
161
|
+
tiledir: MPathLike,
|
|
162
|
+
zoom: Optional[Union[int, List[int]]] = None,
|
|
163
|
+
area: Union[BaseGeometry, str, dict] = None,
|
|
164
|
+
area_crs: Union[CRS, str] = None,
|
|
165
|
+
bounds: Optional[BoundsLike] = None,
|
|
166
|
+
bounds_crs: Union[CRS, str] = None,
|
|
167
|
+
workers: Optional[int] = None,
|
|
168
|
+
) -> List[MPath]:
|
|
169
|
+
return [
|
|
170
|
+
tile
|
|
171
|
+
for tile in gen_existing_paths(
|
|
172
|
+
tiledir=tiledir,
|
|
173
|
+
zoom=zoom,
|
|
174
|
+
area=area,
|
|
175
|
+
area_crs=area_crs,
|
|
176
|
+
bounds=bounds,
|
|
177
|
+
bounds_crs=bounds_crs,
|
|
178
|
+
workers=workers,
|
|
179
|
+
)
|
|
180
|
+
]
|
|
@@ -6,6 +6,7 @@ import warnings
|
|
|
6
6
|
from collections import OrderedDict
|
|
7
7
|
from copy import deepcopy
|
|
8
8
|
from functools import cached_property
|
|
9
|
+
import textwrap
|
|
9
10
|
from typing import Any, Iterator, Optional, Tuple, Union
|
|
10
11
|
|
|
11
12
|
import oyaml as yaml
|
|
@@ -123,7 +124,7 @@ class MapcheteConfig(object):
|
|
|
123
124
|
"""Initialize configuration."""
|
|
124
125
|
# get dictionary representation of input_config and
|
|
125
126
|
# (1) map deprecated params to new structure
|
|
126
|
-
logger.debug(
|
|
127
|
+
logger.debug("parsing %s", input_config)
|
|
127
128
|
try:
|
|
128
129
|
self.parsed_config = parse_config(input_config, strict=stric_parsing)
|
|
129
130
|
self.parsed_config.model_dump()
|
|
@@ -240,9 +241,12 @@ class MapcheteConfig(object):
|
|
|
240
241
|
area_crs=area_crs,
|
|
241
242
|
bounds_crs=bounds_crs,
|
|
242
243
|
)
|
|
243
|
-
logger.info(
|
|
244
|
+
logger.info(
|
|
245
|
+
"process area: %s",
|
|
246
|
+
textwrap.shorten(self.area.wkt, width=200, placeholder="..."),
|
|
247
|
+
)
|
|
244
248
|
self.bounds = Bounds(*self.area.bounds)
|
|
245
|
-
logger.debug(
|
|
249
|
+
logger.debug("process bounds: %s", self.bounds)
|
|
246
250
|
self.init_area = self._get_process_area(
|
|
247
251
|
area=self._init_area,
|
|
248
252
|
bounds=self._init_bounds,
|
|
@@ -251,9 +255,12 @@ class MapcheteConfig(object):
|
|
|
251
255
|
area_crs=area_crs,
|
|
252
256
|
bounds_crs=bounds_crs,
|
|
253
257
|
)
|
|
254
|
-
logger.info(
|
|
258
|
+
logger.info(
|
|
259
|
+
"init area: %s",
|
|
260
|
+
textwrap.shorten(self.init_area.wkt, width=200, placeholder="..."),
|
|
261
|
+
)
|
|
255
262
|
self.init_bounds = Bounds(*self.init_area.bounds)
|
|
256
|
-
logger.debug(
|
|
263
|
+
logger.debug("init bounds: %s", self.init_bounds)
|
|
257
264
|
|
|
258
265
|
# (7) the delimiters are used by some input drivers
|
|
259
266
|
self._delimiters
|
|
@@ -163,8 +163,10 @@ class ConcurrentFuturesExecutor(ExecutorBase):
|
|
|
163
163
|
futures.remove(future)
|
|
164
164
|
|
|
165
165
|
# immediately submit next task from iterator
|
|
166
|
-
|
|
167
|
-
|
|
166
|
+
for item, skip_item, skip_info in item_skip_tuples:
|
|
167
|
+
# yield as many skipped futures as they come but when a future
|
|
168
|
+
# is to be submitted, exit this loop and wait for the next finished
|
|
169
|
+
# futures
|
|
168
170
|
|
|
169
171
|
# skip task submission if option is activated
|
|
170
172
|
if skip_item: # pragma: no cover
|
|
@@ -174,9 +176,11 @@ class ConcurrentFuturesExecutor(ExecutorBase):
|
|
|
174
176
|
else:
|
|
175
177
|
futures.add(self._submit(func, item, fargs, fkwargs))
|
|
176
178
|
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
179
|
+
# jump out of item tuples loop
|
|
180
|
+
break
|
|
181
|
+
|
|
182
|
+
if self.cancel_signal: # pragma: no cover
|
|
183
|
+
raise JobCancelledError("cancel signal caught")
|
|
180
184
|
|
|
181
185
|
if self.cancel_signal: # pragma: no cover
|
|
182
186
|
raise JobCancelledError("cancel signal caught")
|
|
@@ -211,7 +211,7 @@ class InputTile(base.InputTile, VectorInput):
|
|
|
211
211
|
raise NotImplementedError()
|
|
212
212
|
return self._from_cache(validity_check=validity_check)
|
|
213
213
|
|
|
214
|
-
def is_empty(self, validity_check=True): # pragma: no cover
|
|
214
|
+
def is_empty(self, validity_check=True, **_): # pragma: no cover
|
|
215
215
|
"""
|
|
216
216
|
Check if there is data within this tile.
|
|
217
217
|
|
|
@@ -714,7 +714,7 @@ class InputTile(base.InputTile, RasterInput):
|
|
|
714
714
|
else ma.concatenate([ma.expand_dims(arr[i - 1], 0) for i in band_indexes])
|
|
715
715
|
)
|
|
716
716
|
|
|
717
|
-
def is_empty(self,
|
|
717
|
+
def is_empty(self, **_):
|
|
718
718
|
"""
|
|
719
719
|
Check if there is data within this tile.
|
|
720
720
|
|
|
@@ -1,7 +1,11 @@
|
|
|
1
|
+
import logging
|
|
1
2
|
from typing import List
|
|
2
|
-
from shapely import box
|
|
3
|
+
from shapely import box, clip_by_rect
|
|
3
4
|
from shapely.affinity import translate
|
|
5
|
+
from shapely.geometry import GeometryCollection
|
|
6
|
+
|
|
4
7
|
from mapchete.bounds import Bounds
|
|
8
|
+
from mapchete.geometry.repair import repair as repair_geometry
|
|
5
9
|
from mapchete.geometry.shape import to_shape
|
|
6
10
|
from mapchete.geometry.types import MultipartGeometry
|
|
7
11
|
from mapchete.grid import Grid
|
|
@@ -9,6 +13,32 @@ from mapchete.tile import BufferedTilePyramid
|
|
|
9
13
|
from mapchete.geometry.types import Geometry
|
|
10
14
|
|
|
11
15
|
|
|
16
|
+
logger = logging.getLogger(__name__)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def clip_by_bounds(
|
|
20
|
+
geometry: Geometry,
|
|
21
|
+
bounds: Bounds,
|
|
22
|
+
repair: bool = False,
|
|
23
|
+
ignore_errors: bool = False,
|
|
24
|
+
) -> Geometry:
|
|
25
|
+
"""
|
|
26
|
+
Uses shapely.clip_by_rect to quickly clip geometry by bounds.
|
|
27
|
+
|
|
28
|
+
Can also automatically repair output geometry and return an empty GeometryCollection
|
|
29
|
+
if errors occur and the respective flags are activated
|
|
30
|
+
"""
|
|
31
|
+
try:
|
|
32
|
+
clipped = clip_by_rect(geometry, *bounds)
|
|
33
|
+
return repair_geometry(clipped) if repair else clipped
|
|
34
|
+
except Exception as exc:
|
|
35
|
+
if ignore_errors:
|
|
36
|
+
logger.error(exc)
|
|
37
|
+
return GeometryCollection()
|
|
38
|
+
else:
|
|
39
|
+
raise exc
|
|
40
|
+
|
|
41
|
+
|
|
12
42
|
def clip_geometry_to_pyramid_bounds(
|
|
13
43
|
geometry: Geometry, pyramid: BufferedTilePyramid
|
|
14
44
|
) -> List[Geometry]:
|