mapchete 2026.2.2__tar.gz → 2026.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (143) hide show
  1. {mapchete-2026.2.2 → mapchete-2026.4.0}/PKG-INFO +2 -4
  2. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/__init__.py +1 -1
  3. mapchete-2026.4.0/mapchete/cli/mapchete/rm.py +59 -0
  4. mapchete-2026.4.0/mapchete/cli/mpath/rm.py +115 -0
  5. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/cli/mpath/sync.py +11 -2
  6. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/cli/progress_bar.py +3 -1
  7. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/commands/rm.py +76 -41
  8. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/config/base.py +12 -5
  9. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/executor/concurrent_futures.py +9 -5
  10. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/formats/default/_fiona_base.py +1 -1
  11. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/formats/default/gtiff.py +1 -1
  12. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/formats/default/raster_file.py +1 -1
  13. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/formats/default/tile_directory.py +1 -0
  14. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/formats/default/vector_file.py +1 -1
  15. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/formats/protocols.py +1 -1
  16. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/geometry/clip.py +31 -1
  17. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/geometry/shape.py +9 -5
  18. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/io/raster/referenced_raster.py +1 -1
  19. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/io/vector/indexed_features.py +122 -56
  20. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/io/vector/read.py +5 -5
  21. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/path.py +26 -13
  22. {mapchete-2026.2.2 → mapchete-2026.4.0}/pyproject.toml +1 -4
  23. mapchete-2026.2.2/mapchete/cli/mapchete/rm.py +0 -49
  24. mapchete-2026.2.2/mapchete/cli/mpath/rm.py +0 -19
  25. {mapchete-2026.2.2 → mapchete-2026.4.0}/.gitignore +0 -0
  26. {mapchete-2026.2.2 → mapchete-2026.4.0}/LICENSE +0 -0
  27. {mapchete-2026.2.2 → mapchete-2026.4.0}/README.rst +0 -0
  28. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/_deprecated.py +0 -0
  29. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/bounds.py +0 -0
  30. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/cli/__init__.py +0 -0
  31. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/cli/mapchete/__init__.py +0 -0
  32. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/cli/mapchete/convert.py +0 -0
  33. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/cli/mapchete/cp.py +0 -0
  34. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/cli/mapchete/create.py +0 -0
  35. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/cli/mapchete/execute.py +0 -0
  36. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/cli/mapchete/formats.py +0 -0
  37. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/cli/mapchete/index.py +0 -0
  38. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/cli/mapchete/processes.py +0 -0
  39. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/cli/mapchete/serve.py +0 -0
  40. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/cli/mapchete/show_env.py +0 -0
  41. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/cli/mapchete/stac.py +0 -0
  42. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/cli/mpath/__init__.py +0 -0
  43. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/cli/mpath/cp.py +0 -0
  44. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/cli/mpath/exists.py +0 -0
  45. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/cli/mpath/ls.py +0 -0
  46. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/cli/mpath/read_json.py +0 -0
  47. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/cli/mpath/read_text.py +0 -0
  48. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/cli/mpath/read_yaml.py +0 -0
  49. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/cli/options.py +0 -0
  50. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/commands/__init__.py +0 -0
  51. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/commands/convert.py +0 -0
  52. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/commands/cp.py +0 -0
  53. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/commands/execute.py +0 -0
  54. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/commands/index.py +0 -0
  55. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/commands/observer.py +0 -0
  56. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/commands/parser.py +0 -0
  57. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/config/__init__.py +0 -0
  58. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/config/models.py +0 -0
  59. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/config/parse.py +0 -0
  60. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/config/process_func.py +0 -0
  61. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/enums.py +0 -0
  62. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/errors.py +0 -0
  63. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/executor/__init__.py +0 -0
  64. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/executor/base.py +0 -0
  65. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/executor/dask.py +0 -0
  66. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/executor/future.py +0 -0
  67. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/executor/sequential.py +0 -0
  68. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/executor/types.py +0 -0
  69. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/formats/__init__.py +0 -0
  70. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/formats/base.py +0 -0
  71. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/formats/default/__init__.py +0 -0
  72. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/formats/default/flatgeobuf.py +0 -0
  73. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/formats/default/geojson.py +0 -0
  74. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/formats/default/mapchete_input.py +0 -0
  75. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/formats/default/png.py +0 -0
  76. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/formats/default/png_hillshade.py +0 -0
  77. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/formats/drivers.py +0 -0
  78. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/formats/loaders.py +0 -0
  79. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/formats/tools.py +0 -0
  80. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/geometry/__init__.py +0 -0
  81. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/geometry/filter.py +0 -0
  82. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/geometry/footprints.py +0 -0
  83. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/geometry/latlon.py +0 -0
  84. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/geometry/repair.py +0 -0
  85. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/geometry/reproject.py +0 -0
  86. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/geometry/segmentize.py +0 -0
  87. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/geometry/transform.py +0 -0
  88. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/geometry/types.py +0 -0
  89. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/grid.py +0 -0
  90. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/index.py +0 -0
  91. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/io/__init__.py +0 -0
  92. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/io/_json.py +0 -0
  93. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/io/_misc.py +0 -0
  94. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/io/_path.py +0 -0
  95. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/io/profiles.py +0 -0
  96. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/io/raster/__init__.py +0 -0
  97. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/io/raster/array.py +0 -0
  98. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/io/raster/convert.py +0 -0
  99. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/io/raster/mosaic.py +0 -0
  100. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/io/raster/open.py +0 -0
  101. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/io/raster/read.py +0 -0
  102. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/io/raster/write.py +0 -0
  103. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/io/vector/__init__.py +0 -0
  104. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/io/vector/convert.py +0 -0
  105. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/io/vector/open.py +0 -0
  106. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/io/vector/types.py +0 -0
  107. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/io/vector/write.py +0 -0
  108. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/log.py +0 -0
  109. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/pretty.py +0 -0
  110. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/process_func_special_types.py +0 -0
  111. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/processes/__init__.py +0 -0
  112. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/processes/clip.py +0 -0
  113. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/processes/contours.py +0 -0
  114. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/processes/convert.py +0 -0
  115. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/processes/examples/__init__.py +0 -0
  116. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/processes/examples/example_process.py +0 -0
  117. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/processes/hillshade.py +0 -0
  118. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/processing/__init__.py +0 -0
  119. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/processing/base.py +0 -0
  120. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/processing/execute.py +0 -0
  121. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/processing/mp.py +0 -0
  122. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/processing/profilers/__init__.py +0 -0
  123. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/processing/profilers/memory.py +0 -0
  124. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/processing/profilers/requests.py +0 -0
  125. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/processing/profilers/time.py +0 -0
  126. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/processing/tasks.py +0 -0
  127. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/processing/types.py +0 -0
  128. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/protocols.py +0 -0
  129. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/registered.py +0 -0
  130. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/settings.py +0 -0
  131. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/stac/__init__.py +0 -0
  132. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/stac/models.py +0 -0
  133. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/stac/tiled_assets.py +0 -0
  134. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/static/__init__.py +0 -0
  135. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/static/index.html +0 -0
  136. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/static/mapchete_template.mapchete +0 -0
  137. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/static/process_template.py +0 -0
  138. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/testing.py +0 -0
  139. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/tile.py +0 -0
  140. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/timer.py +0 -0
  141. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/types.py +0 -0
  142. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/validate.py +0 -0
  143. {mapchete-2026.2.2 → mapchete-2026.4.0}/mapchete/zoom_levels.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: mapchete
3
- Version: 2026.2.2
3
+ Version: 2026.4.0
4
4
  Summary: Tile-based geodata processing using rasterio & Fiona
5
5
  Project-URL: Homepage, https://github.com/ungarj/mapchete
6
6
  Author-email: Joachim Ungar <joachim.ungar@gmail.com>
@@ -54,7 +54,6 @@ Requires-Dist: matplotlib<3.11; extra == 'complete'
54
54
  Requires-Dist: memray; extra == 'complete'
55
55
  Requires-Dist: pystac[urllib3]>=1.8.2; extra == 'complete'
56
56
  Requires-Dist: requests; extra == 'complete'
57
- Requires-Dist: rtree; extra == 'complete'
58
57
  Requires-Dist: s3fs!=2023.9.0; extra == 'complete'
59
58
  Requires-Dist: tilebench; extra == 'complete'
60
59
  Requires-Dist: werkzeug>=0.15; extra == 'complete'
@@ -76,8 +75,6 @@ Provides-Extra: serve
76
75
  Requires-Dist: flask-rangerequest; extra == 'serve'
77
76
  Requires-Dist: flask<4.0.0; extra == 'serve'
78
77
  Requires-Dist: werkzeug>=0.15; extra == 'serve'
79
- Provides-Extra: spatial-index
80
- Requires-Dist: rtree; extra == 'spatial-index'
81
78
  Provides-Extra: test
82
79
  Requires-Dist: coverage>=5.1; extra == 'test'
83
80
  Requires-Dist: coveralls>=2.2.0; extra == 'test'
@@ -93,6 +90,7 @@ Requires-Dist: pytest-mock; extra == 'test'
93
90
  Requires-Dist: pytest-split; extra == 'test'
94
91
  Requires-Dist: pytest<8,>=5.4.1; extra == 'test'
95
92
  Requires-Dist: rio-cogeo>=5.0.0; extra == 'test'
93
+ Requires-Dist: rtree; extra == 'test'
96
94
  Provides-Extra: vrt
97
95
  Requires-Dist: lxml; extra == 'vrt'
98
96
  Description-Content-Type: text/x-rst
@@ -50,7 +50,7 @@ __all__ = [
50
50
  "VectorInputGroup",
51
51
  "ZoomLevels",
52
52
  ]
53
- __version__ = "2026.2.2"
53
+ __version__ = "2026.4.0"
54
54
 
55
55
  logger = logging.getLogger(__name__)
56
56
  logger.addHandler(logging.NullHandler())
@@ -0,0 +1,59 @@
1
+ import click
2
+ import tqdm
3
+
4
+ from mapchete import commands
5
+ from mapchete.cli import options
6
+ from mapchete.cli.progress_bar import PBar
7
+ from mapchete.commands.rm import existing_paths, gen_existing_paths
8
+
9
+
10
+ @click.command(help="Remove tiles from TileDirectory.")
11
+ @options.arg_tiledir
12
+ @options.opt_zoom
13
+ @options.opt_area
14
+ @options.opt_area_crs
15
+ @options.opt_bounds
16
+ @options.opt_bounds_crs
17
+ @options.opt_workers
18
+ @options.opt_verbose
19
+ @options.opt_no_pbar
20
+ @options.opt_debug
21
+ @options.opt_logfile
22
+ @options.opt_force
23
+ def rm(
24
+ *args,
25
+ force=False,
26
+ debug=False,
27
+ no_pbar=False,
28
+ verbose=False,
29
+ logfile=None,
30
+ **kwargs,
31
+ ):
32
+ """Remove tiles from TileDirectory."""
33
+ if force:
34
+ with PBar(
35
+ desc="tiles",
36
+ disable=debug or no_pbar,
37
+ print_messages=verbose,
38
+ ) as pbar:
39
+ commands.rm(
40
+ paths=gen_existing_paths(*args, **kwargs), observers=[pbar], **kwargs
41
+ )
42
+ else:
43
+ tiles_to_delete = existing_paths(
44
+ *args,
45
+ **kwargs,
46
+ )
47
+ if len(tiles_to_delete):
48
+ if click.confirm(
49
+ f"Do you want to delete {len(tiles_to_delete)} tiles?", abort=True
50
+ ):
51
+ with PBar(
52
+ total=len(tiles_to_delete),
53
+ desc="tiles",
54
+ disable=debug or no_pbar,
55
+ print_messages=verbose,
56
+ ) as pbar:
57
+ commands.rm(paths=tiles_to_delete, observers=[pbar], **kwargs)
58
+ else: # pragma: no cover
59
+ tqdm.tqdm.write("No tiles found to delete.")
@@ -0,0 +1,115 @@
1
+ import logging
2
+ from typing import Optional
3
+
4
+ import click
5
+ import click_spinner
6
+ import tqdm
7
+
8
+ from mapchete.cli import options
9
+ from mapchete.enums import Concurrency
10
+ from mapchete.executor import get_executor
11
+ from mapchete.path import MPath
12
+
13
+
14
+ logger = logging.getLogger(__name__)
15
+
16
+
17
+ @click.command(help="Remove path.")
18
+ @options.arg_path
19
+ @options.opt_src_fs_opts
20
+ @options.opt_recursive
21
+ @options.opt_force
22
+ @options.opt_verbose
23
+ @options.opt_workers
24
+ @click.option(
25
+ "--count",
26
+ is_flag=True,
27
+ help="Count all files from source path. WARNING: this will trigger more requests on S3.",
28
+ )
29
+ def rm(
30
+ path: MPath,
31
+ recursive: bool = False,
32
+ force: bool = False,
33
+ verbose: bool = True,
34
+ count: bool = False,
35
+ debug: bool = False,
36
+ workers: Optional[int] = None,
37
+ **_,
38
+ ):
39
+ if force or click.confirm(f"do you really want to permanently delete {str(path)}?"):
40
+ try:
41
+ if path.is_directory():
42
+ if recursive is False: # pragma: no cover
43
+ raise click.UsageError(
44
+ "--recursive flag has to be active if path is a directory"
45
+ )
46
+
47
+ with get_executor(
48
+ concurrency=Concurrency.none
49
+ if workers == 1
50
+ else Concurrency.threads,
51
+ max_workers=workers,
52
+ ) as executor:
53
+ if count:
54
+ click.echo("counting pages ...")
55
+ with click_spinner.Spinner(disable=debug):
56
+ pages = list(path.paginate())
57
+ click.echo(f"found {len(pages)} pages")
58
+ else:
59
+ pages = path.paginate()
60
+ deleted = 0
61
+ with tqdm.tqdm(
62
+ total=len(pages) if isinstance(pages, list) else None,
63
+ desc="pages",
64
+ ) as pbar:
65
+ for page in pages:
66
+ # when using a generator, manually update total length
67
+ if not isinstance(pages, list):
68
+ pbar.total = (pbar.total or 0) + 1
69
+ pbar.refresh()
70
+
71
+ if verbose: # pragma: no cover
72
+ tqdm.tqdm.write(f"found {len(page)} files")
73
+ logger.debug("found %s files", len(page))
74
+
75
+ for future in tqdm.tqdm(
76
+ executor.as_completed(
77
+ rm_file,
78
+ page,
79
+ max_submitted_tasks=workers * 10,
80
+ ),
81
+ disable=debug,
82
+ total=len(page),
83
+ desc="files",
84
+ leave=False,
85
+ ):
86
+ future.raise_if_failed()
87
+ if verbose:
88
+ msg = future.result()
89
+ tqdm.tqdm.write(msg)
90
+
91
+ deleted += len(page)
92
+ pbar.update()
93
+
94
+ # finally delete folder after all its contents have been deleted
95
+ path.rm(recursive=True)
96
+ click.echo(f"{deleted} files deleted")
97
+
98
+ else:
99
+ msg = rm_file(path)
100
+ if verbose: # pragma: no cover
101
+ tqdm.tqdm.write(msg)
102
+
103
+ except Exception as exc: # pragma: no cover
104
+ raise click.ClickException(str(exc))
105
+
106
+
107
+ def rm_file(file: MPath) -> str:
108
+ try:
109
+ file.rm()
110
+ msg = f"[OK] {str(file)}: deleted"
111
+ logger.debug(msg)
112
+ return msg
113
+ except Exception as exc: # pragma: no cover
114
+ logger.exception(exc)
115
+ raise
@@ -42,6 +42,7 @@ logger = logging.getLogger(__name__)
42
42
  @options.opt_workers
43
43
  @options.opt_debug
44
44
  @options.opt_verbose
45
+ @click.option("--dry-run", is_flag=True, help="Don't copy anything.")
45
46
  def sync(
46
47
  path: MPath,
47
48
  out_path: MPath,
@@ -51,6 +52,7 @@ def sync(
51
52
  workers: int = 1,
52
53
  debug: bool = False,
53
54
  verbose: bool = False,
55
+ dry_run: bool = False,
54
56
  **_,
55
57
  ):
56
58
  try:
@@ -83,7 +85,7 @@ def sync(
83
85
  path, out_path, compare_checksums=compare_checksums
84
86
  ),
85
87
  fargs=None,
86
- fkwargs=dict(chunksize=chunksize, debug=debug),
88
+ fkwargs=dict(chunksize=chunksize, debug=debug, dry_run=dry_run),
87
89
  item_skip_bool=True,
88
90
  max_submitted_tasks=workers * 10,
89
91
  ),
@@ -144,8 +146,15 @@ def check_files(
144
146
 
145
147
 
146
148
  def sync_file(
147
- paths: Tuple[MPath, MPath], chunksize: int = 1024 * 1024, debug: bool = False
149
+ paths: Tuple[MPath, MPath],
150
+ chunksize: int = 1024 * 1024,
151
+ debug: bool = False,
152
+ dry_run: bool = False,
148
153
  ) -> Tuple[Tuple[MPath, MPath], str]:
154
+
155
+ if dry_run: # pragma: no cover
156
+ return paths, "0s [DRY-RUN]"
157
+
149
158
  src_file, dst_file = paths
150
159
  with PBar(
151
160
  total=100,
@@ -38,7 +38,9 @@ class PBar(ObserverProtocol):
38
38
  if self._pbar.total is None or self._pbar.total != progress.total:
39
39
  self._pbar.reset(progress.total) # type: ignore
40
40
 
41
- self._pbar.update(progress.current - self._pbar.n)
41
+ # self._pbar.update(progress.current - self._pbar.n)
42
+ self._pbar.n = progress.current
43
+ self._pbar.update(n=0)
42
44
 
43
45
  if self.print_messages:
44
46
  if task_info:
@@ -1,7 +1,8 @@
1
1
  """Remove tiles from Tile Directory."""
2
2
 
3
+ from itertools import chain, islice
3
4
  import logging
4
- from typing import List, Optional, Union, Dict, Any
5
+ from typing import List, Optional, Union, Dict, Any, Generator, Iterable
5
6
 
6
7
  from rasterio.crs import CRS
7
8
  from shapely.geometry.base import BaseGeometry
@@ -11,14 +12,13 @@ from mapchete.commands.observer import ObserverProtocol, Observers
11
12
  from mapchete.io import tiles_exist
12
13
  from mapchete.path import MPath
13
14
  from mapchete.types import MPathLike, Progress, BoundsLike
14
- from mapchete.tile import BufferedTile
15
15
 
16
16
  logger = logging.getLogger(__name__)
17
17
 
18
18
 
19
19
  def rm(
20
20
  tiledir: Optional[MPathLike] = None,
21
- paths: Optional[List[MPath]] = None,
21
+ paths: Optional[Iterable[MPath]] = None,
22
22
  zoom: Optional[Union[int, List[int]]] = None,
23
23
  area: Union[BaseGeometry, str, dict] = None,
24
24
  area_crs: Union[CRS, str] = None,
@@ -27,6 +27,7 @@ def rm(
27
27
  workers: Optional[int] = None,
28
28
  fs_opts: Optional[Dict[str, Any]] = None,
29
29
  observers: Optional[List[ObserverProtocol]] = None,
30
+ s3_delete_chunksize: int = 1000,
30
31
  ):
31
32
  """
32
33
  Remove tiles from TileDirectory.
@@ -50,13 +51,13 @@ def rm(
50
51
  Configuration options for fsspec filesystem.
51
52
  """
52
53
  all_observers = Observers(observers)
53
-
54
- if tiledir:
54
+ if paths is not None:
55
+ pass
56
+ elif tiledir:
55
57
  if zoom is None: # pragma: no cover
56
58
  raise ValueError("zoom level(s) required")
57
- tiledir = MPath.from_inp(tiledir, storage_options=fs_opts)
58
- paths = existing_paths(
59
- tiledir=tiledir,
59
+ paths = gen_existing_paths(
60
+ tiledir=MPath.from_inp(tiledir, storage_options=fs_opts),
60
61
  zoom=zoom,
61
62
  area=area,
62
63
  area_crs=area_crs,
@@ -64,44 +65,65 @@ def rm(
64
65
  bounds_crs=bounds_crs,
65
66
  workers=workers,
66
67
  )
67
- elif isinstance(paths, list):
68
- pass
69
68
  else: # pragma: no cover
70
- raise ValueError(
71
- "either a tile directory or a list of paths has to be provided"
72
- )
73
-
74
- if not paths:
69
+ raise ValueError("either tiledir or paths have to be provided")
70
+
71
+ try:
72
+ # this only works for lists
73
+ total = len(paths)
74
+ all_observers.notify(progress=Progress(total=total))
75
+ logger.debug("got %s path(s)", len(paths))
76
+ except TypeError:
77
+ # we don't know the length of a generator in advance
78
+ total = None
79
+
80
+ # create an iterator to have a common interface for lists and generators
81
+ paths_iter = iter(paths)
82
+ # determine the filesystem by getting the first path
83
+ try:
84
+ first_path = next(paths_iter)
85
+ except StopIteration:
86
+ # this happends if the list or generator does not have any item
75
87
  logger.debug("no paths to delete")
76
88
  return
77
89
 
78
- all_observers.notify(progress=Progress(total=len(paths)))
79
- logger.debug("got %s path(s)", len(paths))
90
+ fs = first_path.fs
91
+ # append first_path to other paths again
92
+ all_paths = chain((first_path,), paths_iter)
80
93
 
81
94
  # s3fs enables multiple paths as input, so let's use this:
82
- if "s3" in paths[0].protocols:
83
- paths[0].fs.rm(paths)
84
- for ii, path in enumerate(paths, 1):
85
- msg = f"deleted {path}"
86
- logger.debug(msg)
95
+ if "s3" in fs.protocol:
96
+ tiles_counter = 0
97
+ while chunk := tuple(islice(all_paths, s3_delete_chunksize)):
87
98
  all_observers.notify(
88
- progress=Progress(current=ii, total=len(paths)), message=msg
99
+ progress=Progress(
100
+ current=tiles_counter, total=total or tiles_counter + len(chunk)
101
+ )
89
102
  )
103
+ # this actually deletes the files
104
+ fs.rm(chunk)
105
+ for path in chunk:
106
+ tiles_counter += 1
107
+ msg = f"deleted {path}"
108
+ logger.debug(msg)
109
+ all_observers.notify(message=msg)
110
+ all_observers.notify(progress=Progress(current=tiles_counter))
90
111
 
91
112
  # otherwise, just iterate through the paths
92
113
  else:
93
- for ii, path in enumerate(paths, 1):
114
+ for tiles_counter, path in enumerate(all_paths, 1):
94
115
  path.rm()
95
116
  msg = f"deleted {path}"
96
117
  logger.debug(msg)
97
118
  all_observers.notify(
98
- progress=Progress(current=ii, total=len(paths)), message=msg
119
+ progress=Progress(current=tiles_counter, total=total or tiles_counter),
120
+ message=msg,
99
121
  )
100
122
 
101
- all_observers.notify(message=f"{len(paths)} tiles deleted")
123
+ all_observers.notify(message=f"{tiles_counter} tiles deleted")
102
124
 
103
125
 
104
- def existing_paths(
126
+ def gen_existing_paths(
105
127
  tiledir: MPathLike,
106
128
  zoom: Optional[Union[int, List[int]]] = None,
107
129
  area: Union[BaseGeometry, str, dict] = None,
@@ -109,7 +131,7 @@ def existing_paths(
109
131
  bounds: Optional[BoundsLike] = None,
110
132
  bounds_crs: Union[CRS, str] = None,
111
133
  workers: Optional[int] = None,
112
- ) -> List[MPath]:
134
+ ) -> Generator[MPath, None, None]:
113
135
  with mapchete.open(
114
136
  tiledir,
115
137
  zoom=zoom,
@@ -120,26 +142,39 @@ def existing_paths(
120
142
  mode="readonly",
121
143
  ) as mp:
122
144
  tp = mp.config.output_pyramid
123
- tiles: Dict[Union[int, List[int], None], List[BufferedTile]] = {}
124
145
  for zoom in mp.config.init_zoom_levels:
125
- tiles[zoom] = []
126
146
  # check which source tiles exist
127
147
  logger.debug("looking for existing source tiles in zoom %s...", zoom)
128
148
  for tile, exists in tiles_exist(
129
149
  config=mp.config,
130
- output_tiles=(
131
- tile_
132
- for tile_ in tp.tiles_from_geom(mp.config.area_at_zoom(zoom), zoom)
133
- # this is required to omit tiles touching the config area
134
- if mp.config.area_at_zoom(zoom).intersection(tile_.bbox).area
150
+ output_tiles_batches=tp.tiles_from_geom_batches(
151
+ mp.config.area_at_zoom(zoom), zoom
135
152
  ),
136
153
  workers=workers,
137
154
  ):
138
155
  if exists:
139
- tiles[zoom].append(tile)
156
+ logger.debug("yield tile %s", tile)
157
+ yield mp.config.output_reader.get_path(tile)
158
+
140
159
 
141
- return [
142
- mp.config.output_reader.get_path(tile)
143
- for zoom_tiles in tiles.values()
144
- for tile in zoom_tiles
145
- ]
160
+ def existing_paths(
161
+ tiledir: MPathLike,
162
+ zoom: Optional[Union[int, List[int]]] = None,
163
+ area: Union[BaseGeometry, str, dict] = None,
164
+ area_crs: Union[CRS, str] = None,
165
+ bounds: Optional[BoundsLike] = None,
166
+ bounds_crs: Union[CRS, str] = None,
167
+ workers: Optional[int] = None,
168
+ ) -> List[MPath]:
169
+ return [
170
+ tile
171
+ for tile in gen_existing_paths(
172
+ tiledir=tiledir,
173
+ zoom=zoom,
174
+ area=area,
175
+ area_crs=area_crs,
176
+ bounds=bounds,
177
+ bounds_crs=bounds_crs,
178
+ workers=workers,
179
+ )
180
+ ]
@@ -6,6 +6,7 @@ import warnings
6
6
  from collections import OrderedDict
7
7
  from copy import deepcopy
8
8
  from functools import cached_property
9
+ import textwrap
9
10
  from typing import Any, Iterator, Optional, Tuple, Union
10
11
 
11
12
  import oyaml as yaml
@@ -123,7 +124,7 @@ class MapcheteConfig(object):
123
124
  """Initialize configuration."""
124
125
  # get dictionary representation of input_config and
125
126
  # (1) map deprecated params to new structure
126
- logger.debug(f"parsing {input_config}")
127
+ logger.debug("parsing %s", input_config)
127
128
  try:
128
129
  self.parsed_config = parse_config(input_config, strict=stric_parsing)
129
130
  self.parsed_config.model_dump()
@@ -240,9 +241,12 @@ class MapcheteConfig(object):
240
241
  area_crs=area_crs,
241
242
  bounds_crs=bounds_crs,
242
243
  )
243
- logger.info(f"process area: {self.area}")
244
+ logger.info(
245
+ "process area: %s",
246
+ textwrap.shorten(self.area.wkt, width=200, placeholder="..."),
247
+ )
244
248
  self.bounds = Bounds(*self.area.bounds)
245
- logger.debug(f"process bounds: {self.bounds}")
249
+ logger.debug("process bounds: %s", self.bounds)
246
250
  self.init_area = self._get_process_area(
247
251
  area=self._init_area,
248
252
  bounds=self._init_bounds,
@@ -251,9 +255,12 @@ class MapcheteConfig(object):
251
255
  area_crs=area_crs,
252
256
  bounds_crs=bounds_crs,
253
257
  )
254
- logger.info(f"init area: {self.init_area}")
258
+ logger.info(
259
+ "init area: %s",
260
+ textwrap.shorten(self.init_area.wkt, width=200, placeholder="..."),
261
+ )
255
262
  self.init_bounds = Bounds(*self.init_area.bounds)
256
- logger.debug(f"init bounds: {self.init_bounds}")
263
+ logger.debug("init bounds: %s", self.init_bounds)
257
264
 
258
265
  # (7) the delimiters are used by some input drivers
259
266
  self._delimiters
@@ -163,8 +163,10 @@ class ConcurrentFuturesExecutor(ExecutorBase):
163
163
  futures.remove(future)
164
164
 
165
165
  # immediately submit next task from iterator
166
- try:
167
- item, skip_item, skip_info = next(item_skip_tuples)
166
+ for item, skip_item, skip_info in item_skip_tuples:
167
+ # yield as many skipped futures as they come but when a future
168
+ # is to be submitted, exit this loop and wait for the next finished
169
+ # futures
168
170
 
169
171
  # skip task submission if option is activated
170
172
  if skip_item: # pragma: no cover
@@ -174,9 +176,11 @@ class ConcurrentFuturesExecutor(ExecutorBase):
174
176
  else:
175
177
  futures.add(self._submit(func, item, fargs, fkwargs))
176
178
 
177
- except StopIteration:
178
- # nothing left to submit
179
- pass
179
+ # jump out of item tuples loop
180
+ break
181
+
182
+ if self.cancel_signal: # pragma: no cover
183
+ raise JobCancelledError("cancel signal caught")
180
184
 
181
185
  if self.cancel_signal: # pragma: no cover
182
186
  raise JobCancelledError("cancel signal caught")
@@ -211,7 +211,7 @@ class InputTile(base.InputTile, VectorInput):
211
211
  raise NotImplementedError()
212
212
  return self._from_cache(validity_check=validity_check)
213
213
 
214
- def is_empty(self, validity_check=True): # pragma: no cover
214
+ def is_empty(self, validity_check=True, **_): # pragma: no cover
215
215
  """
216
216
  Check if there is data within this tile.
217
217
 
@@ -714,7 +714,7 @@ class InputTile(base.InputTile, RasterInput):
714
714
  else ma.concatenate([ma.expand_dims(arr[i - 1], 0) for i in band_indexes])
715
715
  )
716
716
 
717
- def is_empty(self, indexes=None):
717
+ def is_empty(self, **_):
718
718
  """
719
719
  Check if there is data within this tile.
720
720
 
@@ -312,7 +312,7 @@ class InputTile(base.InputTile, RasterInput):
312
312
  gdal_opts=self.gdal_opts,
313
313
  )
314
314
 
315
- def is_empty(self, indexes=None):
315
+ def is_empty(self, **_):
316
316
  """
317
317
  Check if there is data within this tile.
318
318
 
@@ -358,6 +358,7 @@ class InputTile(base.InputTile, RasterInput):
358
358
  matching_method="gdal",
359
359
  matching_precision=8,
360
360
  matching_max_zoom=None,
361
+ **_,
361
362
  ):
362
363
  """
363
364
  Check if there is data within this tile.
@@ -328,7 +328,7 @@ class InputTile(base.InputTile, VectorInput):
328
328
  target_geometry_type=target_geometry_type,
329
329
  )
330
330
 
331
- def is_empty(self) -> bool:
331
+ def is_empty(self, **_) -> bool:
332
332
  """
333
333
  Check if there is data within this tile.
334
334
 
@@ -29,7 +29,7 @@ class InputTileProtocol(GridProtocol): # pragma: no cover
29
29
  """Read from input."""
30
30
  ...
31
31
 
32
- def is_empty(self) -> bool:
32
+ def is_empty(self, **kwargs) -> bool:
33
33
  """Checks if input is empty here."""
34
34
  ...
35
35
 
@@ -1,7 +1,11 @@
1
+ import logging
1
2
  from typing import List
2
- from shapely import box
3
+ from shapely import box, clip_by_rect
3
4
  from shapely.affinity import translate
5
+ from shapely.geometry import GeometryCollection
6
+
4
7
  from mapchete.bounds import Bounds
8
+ from mapchete.geometry.repair import repair as repair_geometry
5
9
  from mapchete.geometry.shape import to_shape
6
10
  from mapchete.geometry.types import MultipartGeometry
7
11
  from mapchete.grid import Grid
@@ -9,6 +13,32 @@ from mapchete.tile import BufferedTilePyramid
9
13
  from mapchete.geometry.types import Geometry
10
14
 
11
15
 
16
+ logger = logging.getLogger(__name__)
17
+
18
+
19
+ def clip_by_bounds(
20
+ geometry: Geometry,
21
+ bounds: Bounds,
22
+ repair: bool = False,
23
+ ignore_errors: bool = False,
24
+ ) -> Geometry:
25
+ """
26
+ Uses shapely.clip_by_rect to quickly clip geometry by bounds.
27
+
28
+ Can also automatically repair output geometry and return an empty GeometryCollection
29
+ if errors occur and the respective flags are activated
30
+ """
31
+ try:
32
+ clipped = clip_by_rect(geometry, *bounds)
33
+ return repair_geometry(clipped) if repair else clipped
34
+ except Exception as exc:
35
+ if ignore_errors:
36
+ logger.error(exc)
37
+ return GeometryCollection()
38
+ else:
39
+ raise exc
40
+
41
+
12
42
  def clip_geometry_to_pyramid_bounds(
13
43
  geometry: Geometry, pyramid: BufferedTilePyramid
14
44
  ) -> List[Geometry]: