mapchete 2026.2.2__tar.gz → 2026.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (142) hide show
  1. {mapchete-2026.2.2 → mapchete-2026.3.0}/PKG-INFO +2 -4
  2. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/__init__.py +1 -1
  3. mapchete-2026.3.0/mapchete/cli/mapchete/rm.py +59 -0
  4. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/cli/progress_bar.py +3 -1
  5. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/commands/rm.py +76 -41
  6. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/config/base.py +12 -5
  7. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/formats/default/_fiona_base.py +1 -1
  8. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/formats/default/gtiff.py +1 -1
  9. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/formats/default/raster_file.py +1 -1
  10. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/formats/default/tile_directory.py +1 -0
  11. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/formats/default/vector_file.py +1 -1
  12. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/formats/protocols.py +1 -1
  13. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/io/raster/referenced_raster.py +1 -1
  14. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/io/vector/indexed_features.py +122 -56
  15. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/path.py +26 -13
  16. {mapchete-2026.2.2 → mapchete-2026.3.0}/pyproject.toml +1 -4
  17. mapchete-2026.2.2/mapchete/cli/mapchete/rm.py +0 -49
  18. {mapchete-2026.2.2 → mapchete-2026.3.0}/.gitignore +0 -0
  19. {mapchete-2026.2.2 → mapchete-2026.3.0}/LICENSE +0 -0
  20. {mapchete-2026.2.2 → mapchete-2026.3.0}/README.rst +0 -0
  21. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/_deprecated.py +0 -0
  22. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/bounds.py +0 -0
  23. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/cli/__init__.py +0 -0
  24. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/cli/mapchete/__init__.py +0 -0
  25. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/cli/mapchete/convert.py +0 -0
  26. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/cli/mapchete/cp.py +0 -0
  27. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/cli/mapchete/create.py +0 -0
  28. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/cli/mapchete/execute.py +0 -0
  29. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/cli/mapchete/formats.py +0 -0
  30. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/cli/mapchete/index.py +0 -0
  31. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/cli/mapchete/processes.py +0 -0
  32. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/cli/mapchete/serve.py +0 -0
  33. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/cli/mapchete/show_env.py +0 -0
  34. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/cli/mapchete/stac.py +0 -0
  35. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/cli/mpath/__init__.py +0 -0
  36. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/cli/mpath/cp.py +0 -0
  37. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/cli/mpath/exists.py +0 -0
  38. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/cli/mpath/ls.py +0 -0
  39. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/cli/mpath/read_json.py +0 -0
  40. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/cli/mpath/read_text.py +0 -0
  41. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/cli/mpath/read_yaml.py +0 -0
  42. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/cli/mpath/rm.py +0 -0
  43. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/cli/mpath/sync.py +0 -0
  44. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/cli/options.py +0 -0
  45. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/commands/__init__.py +0 -0
  46. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/commands/convert.py +0 -0
  47. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/commands/cp.py +0 -0
  48. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/commands/execute.py +0 -0
  49. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/commands/index.py +0 -0
  50. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/commands/observer.py +0 -0
  51. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/commands/parser.py +0 -0
  52. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/config/__init__.py +0 -0
  53. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/config/models.py +0 -0
  54. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/config/parse.py +0 -0
  55. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/config/process_func.py +0 -0
  56. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/enums.py +0 -0
  57. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/errors.py +0 -0
  58. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/executor/__init__.py +0 -0
  59. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/executor/base.py +0 -0
  60. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/executor/concurrent_futures.py +0 -0
  61. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/executor/dask.py +0 -0
  62. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/executor/future.py +0 -0
  63. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/executor/sequential.py +0 -0
  64. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/executor/types.py +0 -0
  65. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/formats/__init__.py +0 -0
  66. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/formats/base.py +0 -0
  67. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/formats/default/__init__.py +0 -0
  68. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/formats/default/flatgeobuf.py +0 -0
  69. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/formats/default/geojson.py +0 -0
  70. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/formats/default/mapchete_input.py +0 -0
  71. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/formats/default/png.py +0 -0
  72. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/formats/default/png_hillshade.py +0 -0
  73. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/formats/drivers.py +0 -0
  74. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/formats/loaders.py +0 -0
  75. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/formats/tools.py +0 -0
  76. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/geometry/__init__.py +0 -0
  77. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/geometry/clip.py +0 -0
  78. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/geometry/filter.py +0 -0
  79. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/geometry/footprints.py +0 -0
  80. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/geometry/latlon.py +0 -0
  81. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/geometry/repair.py +0 -0
  82. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/geometry/reproject.py +0 -0
  83. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/geometry/segmentize.py +0 -0
  84. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/geometry/shape.py +0 -0
  85. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/geometry/transform.py +0 -0
  86. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/geometry/types.py +0 -0
  87. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/grid.py +0 -0
  88. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/index.py +0 -0
  89. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/io/__init__.py +0 -0
  90. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/io/_json.py +0 -0
  91. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/io/_misc.py +0 -0
  92. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/io/_path.py +0 -0
  93. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/io/profiles.py +0 -0
  94. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/io/raster/__init__.py +0 -0
  95. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/io/raster/array.py +0 -0
  96. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/io/raster/convert.py +0 -0
  97. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/io/raster/mosaic.py +0 -0
  98. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/io/raster/open.py +0 -0
  99. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/io/raster/read.py +0 -0
  100. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/io/raster/write.py +0 -0
  101. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/io/vector/__init__.py +0 -0
  102. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/io/vector/convert.py +0 -0
  103. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/io/vector/open.py +0 -0
  104. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/io/vector/read.py +0 -0
  105. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/io/vector/types.py +0 -0
  106. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/io/vector/write.py +0 -0
  107. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/log.py +0 -0
  108. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/pretty.py +0 -0
  109. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/process_func_special_types.py +0 -0
  110. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/processes/__init__.py +0 -0
  111. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/processes/clip.py +0 -0
  112. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/processes/contours.py +0 -0
  113. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/processes/convert.py +0 -0
  114. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/processes/examples/__init__.py +0 -0
  115. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/processes/examples/example_process.py +0 -0
  116. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/processes/hillshade.py +0 -0
  117. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/processing/__init__.py +0 -0
  118. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/processing/base.py +0 -0
  119. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/processing/execute.py +0 -0
  120. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/processing/mp.py +0 -0
  121. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/processing/profilers/__init__.py +0 -0
  122. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/processing/profilers/memory.py +0 -0
  123. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/processing/profilers/requests.py +0 -0
  124. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/processing/profilers/time.py +0 -0
  125. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/processing/tasks.py +0 -0
  126. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/processing/types.py +0 -0
  127. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/protocols.py +0 -0
  128. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/registered.py +0 -0
  129. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/settings.py +0 -0
  130. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/stac/__init__.py +0 -0
  131. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/stac/models.py +0 -0
  132. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/stac/tiled_assets.py +0 -0
  133. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/static/__init__.py +0 -0
  134. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/static/index.html +0 -0
  135. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/static/mapchete_template.mapchete +0 -0
  136. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/static/process_template.py +0 -0
  137. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/testing.py +0 -0
  138. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/tile.py +0 -0
  139. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/timer.py +0 -0
  140. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/types.py +0 -0
  141. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/validate.py +0 -0
  142. {mapchete-2026.2.2 → mapchete-2026.3.0}/mapchete/zoom_levels.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: mapchete
3
- Version: 2026.2.2
3
+ Version: 2026.3.0
4
4
  Summary: Tile-based geodata processing using rasterio & Fiona
5
5
  Project-URL: Homepage, https://github.com/ungarj/mapchete
6
6
  Author-email: Joachim Ungar <joachim.ungar@gmail.com>
@@ -54,7 +54,6 @@ Requires-Dist: matplotlib<3.11; extra == 'complete'
54
54
  Requires-Dist: memray; extra == 'complete'
55
55
  Requires-Dist: pystac[urllib3]>=1.8.2; extra == 'complete'
56
56
  Requires-Dist: requests; extra == 'complete'
57
- Requires-Dist: rtree; extra == 'complete'
58
57
  Requires-Dist: s3fs!=2023.9.0; extra == 'complete'
59
58
  Requires-Dist: tilebench; extra == 'complete'
60
59
  Requires-Dist: werkzeug>=0.15; extra == 'complete'
@@ -76,8 +75,6 @@ Provides-Extra: serve
76
75
  Requires-Dist: flask-rangerequest; extra == 'serve'
77
76
  Requires-Dist: flask<4.0.0; extra == 'serve'
78
77
  Requires-Dist: werkzeug>=0.15; extra == 'serve'
79
- Provides-Extra: spatial-index
80
- Requires-Dist: rtree; extra == 'spatial-index'
81
78
  Provides-Extra: test
82
79
  Requires-Dist: coverage>=5.1; extra == 'test'
83
80
  Requires-Dist: coveralls>=2.2.0; extra == 'test'
@@ -93,6 +90,7 @@ Requires-Dist: pytest-mock; extra == 'test'
93
90
  Requires-Dist: pytest-split; extra == 'test'
94
91
  Requires-Dist: pytest<8,>=5.4.1; extra == 'test'
95
92
  Requires-Dist: rio-cogeo>=5.0.0; extra == 'test'
93
+ Requires-Dist: rtree; extra == 'test'
96
94
  Provides-Extra: vrt
97
95
  Requires-Dist: lxml; extra == 'vrt'
98
96
  Description-Content-Type: text/x-rst
@@ -50,7 +50,7 @@ __all__ = [
50
50
  "VectorInputGroup",
51
51
  "ZoomLevels",
52
52
  ]
53
- __version__ = "2026.2.2"
53
+ __version__ = "2026.3.0"
54
54
 
55
55
  logger = logging.getLogger(__name__)
56
56
  logger.addHandler(logging.NullHandler())
@@ -0,0 +1,59 @@
1
+ import click
2
+ import tqdm
3
+
4
+ from mapchete import commands
5
+ from mapchete.cli import options
6
+ from mapchete.cli.progress_bar import PBar
7
+ from mapchete.commands.rm import existing_paths, gen_existing_paths
8
+
9
+
10
+ @click.command(help="Remove tiles from TileDirectory.")
11
+ @options.arg_tiledir
12
+ @options.opt_zoom
13
+ @options.opt_area
14
+ @options.opt_area_crs
15
+ @options.opt_bounds
16
+ @options.opt_bounds_crs
17
+ @options.opt_workers
18
+ @options.opt_verbose
19
+ @options.opt_no_pbar
20
+ @options.opt_debug
21
+ @options.opt_logfile
22
+ @options.opt_force
23
+ def rm(
24
+ *args,
25
+ force=False,
26
+ debug=False,
27
+ no_pbar=False,
28
+ verbose=False,
29
+ logfile=None,
30
+ **kwargs,
31
+ ):
32
+ """Remove tiles from TileDirectory."""
33
+ if force:
34
+ with PBar(
35
+ desc="tiles",
36
+ disable=debug or no_pbar,
37
+ print_messages=verbose,
38
+ ) as pbar:
39
+ commands.rm(
40
+ paths=gen_existing_paths(*args, **kwargs), observers=[pbar], **kwargs
41
+ )
42
+ else:
43
+ tiles_to_delete = existing_paths(
44
+ *args,
45
+ **kwargs,
46
+ )
47
+ if len(tiles_to_delete):
48
+ if click.confirm(
49
+ f"Do you want to delete {len(tiles_to_delete)} tiles?", abort=True
50
+ ):
51
+ with PBar(
52
+ total=len(tiles_to_delete),
53
+ desc="tiles",
54
+ disable=debug or no_pbar,
55
+ print_messages=verbose,
56
+ ) as pbar:
57
+ commands.rm(paths=tiles_to_delete, observers=[pbar], **kwargs)
58
+ else: # pragma: no cover
59
+ tqdm.tqdm.write("No tiles found to delete.")
@@ -38,7 +38,9 @@ class PBar(ObserverProtocol):
38
38
  if self._pbar.total is None or self._pbar.total != progress.total:
39
39
  self._pbar.reset(progress.total) # type: ignore
40
40
 
41
- self._pbar.update(progress.current - self._pbar.n)
41
+ # self._pbar.update(progress.current - self._pbar.n)
42
+ self._pbar.n = progress.current
43
+ self._pbar.update(n=0)
42
44
 
43
45
  if self.print_messages:
44
46
  if task_info:
@@ -1,7 +1,8 @@
1
1
  """Remove tiles from Tile Directory."""
2
2
 
3
+ from itertools import chain, islice
3
4
  import logging
4
- from typing import List, Optional, Union, Dict, Any
5
+ from typing import List, Optional, Union, Dict, Any, Generator, Iterable
5
6
 
6
7
  from rasterio.crs import CRS
7
8
  from shapely.geometry.base import BaseGeometry
@@ -11,14 +12,13 @@ from mapchete.commands.observer import ObserverProtocol, Observers
11
12
  from mapchete.io import tiles_exist
12
13
  from mapchete.path import MPath
13
14
  from mapchete.types import MPathLike, Progress, BoundsLike
14
- from mapchete.tile import BufferedTile
15
15
 
16
16
  logger = logging.getLogger(__name__)
17
17
 
18
18
 
19
19
  def rm(
20
20
  tiledir: Optional[MPathLike] = None,
21
- paths: Optional[List[MPath]] = None,
21
+ paths: Optional[Iterable[MPath]] = None,
22
22
  zoom: Optional[Union[int, List[int]]] = None,
23
23
  area: Union[BaseGeometry, str, dict] = None,
24
24
  area_crs: Union[CRS, str] = None,
@@ -27,6 +27,7 @@ def rm(
27
27
  workers: Optional[int] = None,
28
28
  fs_opts: Optional[Dict[str, Any]] = None,
29
29
  observers: Optional[List[ObserverProtocol]] = None,
30
+ s3_delete_chunksize: int = 1000,
30
31
  ):
31
32
  """
32
33
  Remove tiles from TileDirectory.
@@ -50,13 +51,13 @@ def rm(
50
51
  Configuration options for fsspec filesystem.
51
52
  """
52
53
  all_observers = Observers(observers)
53
-
54
- if tiledir:
54
+ if paths is not None:
55
+ pass
56
+ elif tiledir:
55
57
  if zoom is None: # pragma: no cover
56
58
  raise ValueError("zoom level(s) required")
57
- tiledir = MPath.from_inp(tiledir, storage_options=fs_opts)
58
- paths = existing_paths(
59
- tiledir=tiledir,
59
+ paths = gen_existing_paths(
60
+ tiledir=MPath.from_inp(tiledir, storage_options=fs_opts),
60
61
  zoom=zoom,
61
62
  area=area,
62
63
  area_crs=area_crs,
@@ -64,44 +65,65 @@ def rm(
64
65
  bounds_crs=bounds_crs,
65
66
  workers=workers,
66
67
  )
67
- elif isinstance(paths, list):
68
- pass
69
68
  else: # pragma: no cover
70
- raise ValueError(
71
- "either a tile directory or a list of paths has to be provided"
72
- )
73
-
74
- if not paths:
69
+ raise ValueError("either tiledir or paths have to be provided")
70
+
71
+ try:
72
+ # this only works for lists
73
+ total = len(paths)
74
+ all_observers.notify(progress=Progress(total=total))
75
+ logger.debug("got %s path(s)", len(paths))
76
+ except TypeError:
77
+ # we don't know the length of a generator in advance
78
+ total = None
79
+
80
+ # create an iterator to have a common interface for lists and generators
81
+ paths_iter = iter(paths)
82
+ # determine the filesystem by getting the first path
83
+ try:
84
+ first_path = next(paths_iter)
85
+ except StopIteration:
86
+ # this happends if the list or generator does not have any item
75
87
  logger.debug("no paths to delete")
76
88
  return
77
89
 
78
- all_observers.notify(progress=Progress(total=len(paths)))
79
- logger.debug("got %s path(s)", len(paths))
90
+ fs = first_path.fs
91
+ # append first_path to other paths again
92
+ all_paths = chain((first_path,), paths_iter)
80
93
 
81
94
  # s3fs enables multiple paths as input, so let's use this:
82
- if "s3" in paths[0].protocols:
83
- paths[0].fs.rm(paths)
84
- for ii, path in enumerate(paths, 1):
85
- msg = f"deleted {path}"
86
- logger.debug(msg)
95
+ if "s3" in fs.protocol:
96
+ tiles_counter = 0
97
+ while chunk := tuple(islice(all_paths, s3_delete_chunksize)):
87
98
  all_observers.notify(
88
- progress=Progress(current=ii, total=len(paths)), message=msg
99
+ progress=Progress(
100
+ current=tiles_counter, total=total or tiles_counter + len(chunk)
101
+ )
89
102
  )
103
+ # this actually deletes the files
104
+ fs.rm(chunk)
105
+ for path in chunk:
106
+ tiles_counter += 1
107
+ msg = f"deleted {path}"
108
+ logger.debug(msg)
109
+ all_observers.notify(message=msg)
110
+ all_observers.notify(progress=Progress(current=tiles_counter))
90
111
 
91
112
  # otherwise, just iterate through the paths
92
113
  else:
93
- for ii, path in enumerate(paths, 1):
114
+ for tiles_counter, path in enumerate(all_paths, 1):
94
115
  path.rm()
95
116
  msg = f"deleted {path}"
96
117
  logger.debug(msg)
97
118
  all_observers.notify(
98
- progress=Progress(current=ii, total=len(paths)), message=msg
119
+ progress=Progress(current=tiles_counter, total=total or tiles_counter),
120
+ message=msg,
99
121
  )
100
122
 
101
- all_observers.notify(message=f"{len(paths)} tiles deleted")
123
+ all_observers.notify(message=f"{tiles_counter} tiles deleted")
102
124
 
103
125
 
104
- def existing_paths(
126
+ def gen_existing_paths(
105
127
  tiledir: MPathLike,
106
128
  zoom: Optional[Union[int, List[int]]] = None,
107
129
  area: Union[BaseGeometry, str, dict] = None,
@@ -109,7 +131,7 @@ def existing_paths(
109
131
  bounds: Optional[BoundsLike] = None,
110
132
  bounds_crs: Union[CRS, str] = None,
111
133
  workers: Optional[int] = None,
112
- ) -> List[MPath]:
134
+ ) -> Generator[MPath, None, None]:
113
135
  with mapchete.open(
114
136
  tiledir,
115
137
  zoom=zoom,
@@ -120,26 +142,39 @@ def existing_paths(
120
142
  mode="readonly",
121
143
  ) as mp:
122
144
  tp = mp.config.output_pyramid
123
- tiles: Dict[Union[int, List[int], None], List[BufferedTile]] = {}
124
145
  for zoom in mp.config.init_zoom_levels:
125
- tiles[zoom] = []
126
146
  # check which source tiles exist
127
147
  logger.debug("looking for existing source tiles in zoom %s...", zoom)
128
148
  for tile, exists in tiles_exist(
129
149
  config=mp.config,
130
- output_tiles=(
131
- tile_
132
- for tile_ in tp.tiles_from_geom(mp.config.area_at_zoom(zoom), zoom)
133
- # this is required to omit tiles touching the config area
134
- if mp.config.area_at_zoom(zoom).intersection(tile_.bbox).area
150
+ output_tiles_batches=tp.tiles_from_geom_batches(
151
+ mp.config.area_at_zoom(zoom), zoom
135
152
  ),
136
153
  workers=workers,
137
154
  ):
138
155
  if exists:
139
- tiles[zoom].append(tile)
156
+ logger.debug("yield tile %s", tile)
157
+ yield mp.config.output_reader.get_path(tile)
158
+
140
159
 
141
- return [
142
- mp.config.output_reader.get_path(tile)
143
- for zoom_tiles in tiles.values()
144
- for tile in zoom_tiles
145
- ]
160
+ def existing_paths(
161
+ tiledir: MPathLike,
162
+ zoom: Optional[Union[int, List[int]]] = None,
163
+ area: Union[BaseGeometry, str, dict] = None,
164
+ area_crs: Union[CRS, str] = None,
165
+ bounds: Optional[BoundsLike] = None,
166
+ bounds_crs: Union[CRS, str] = None,
167
+ workers: Optional[int] = None,
168
+ ) -> List[MPath]:
169
+ return [
170
+ tile
171
+ for tile in gen_existing_paths(
172
+ tiledir=tiledir,
173
+ zoom=zoom,
174
+ area=area,
175
+ area_crs=area_crs,
176
+ bounds=bounds,
177
+ bounds_crs=bounds_crs,
178
+ workers=workers,
179
+ )
180
+ ]
@@ -6,6 +6,7 @@ import warnings
6
6
  from collections import OrderedDict
7
7
  from copy import deepcopy
8
8
  from functools import cached_property
9
+ import textwrap
9
10
  from typing import Any, Iterator, Optional, Tuple, Union
10
11
 
11
12
  import oyaml as yaml
@@ -123,7 +124,7 @@ class MapcheteConfig(object):
123
124
  """Initialize configuration."""
124
125
  # get dictionary representation of input_config and
125
126
  # (1) map deprecated params to new structure
126
- logger.debug(f"parsing {input_config}")
127
+ logger.debug("parsing %s", input_config)
127
128
  try:
128
129
  self.parsed_config = parse_config(input_config, strict=stric_parsing)
129
130
  self.parsed_config.model_dump()
@@ -240,9 +241,12 @@ class MapcheteConfig(object):
240
241
  area_crs=area_crs,
241
242
  bounds_crs=bounds_crs,
242
243
  )
243
- logger.info(f"process area: {self.area}")
244
+ logger.info(
245
+ "process area: %s",
246
+ textwrap.shorten(self.area.wkt, width=200, placeholder="..."),
247
+ )
244
248
  self.bounds = Bounds(*self.area.bounds)
245
- logger.debug(f"process bounds: {self.bounds}")
249
+ logger.debug("process bounds: %s", self.bounds)
246
250
  self.init_area = self._get_process_area(
247
251
  area=self._init_area,
248
252
  bounds=self._init_bounds,
@@ -251,9 +255,12 @@ class MapcheteConfig(object):
251
255
  area_crs=area_crs,
252
256
  bounds_crs=bounds_crs,
253
257
  )
254
- logger.info(f"init area: {self.init_area}")
258
+ logger.info(
259
+ "init area: %s",
260
+ textwrap.shorten(self.init_area.wkt, width=200, placeholder="..."),
261
+ )
255
262
  self.init_bounds = Bounds(*self.init_area.bounds)
256
- logger.debug(f"init bounds: {self.init_bounds}")
263
+ logger.debug("init bounds: %s", self.init_bounds)
257
264
 
258
265
  # (7) the delimiters are used by some input drivers
259
266
  self._delimiters
@@ -211,7 +211,7 @@ class InputTile(base.InputTile, VectorInput):
211
211
  raise NotImplementedError()
212
212
  return self._from_cache(validity_check=validity_check)
213
213
 
214
- def is_empty(self, validity_check=True): # pragma: no cover
214
+ def is_empty(self, validity_check=True, **_): # pragma: no cover
215
215
  """
216
216
  Check if there is data within this tile.
217
217
 
@@ -714,7 +714,7 @@ class InputTile(base.InputTile, RasterInput):
714
714
  else ma.concatenate([ma.expand_dims(arr[i - 1], 0) for i in band_indexes])
715
715
  )
716
716
 
717
- def is_empty(self, indexes=None):
717
+ def is_empty(self, **_):
718
718
  """
719
719
  Check if there is data within this tile.
720
720
 
@@ -312,7 +312,7 @@ class InputTile(base.InputTile, RasterInput):
312
312
  gdal_opts=self.gdal_opts,
313
313
  )
314
314
 
315
- def is_empty(self, indexes=None):
315
+ def is_empty(self, **_):
316
316
  """
317
317
  Check if there is data within this tile.
318
318
 
@@ -358,6 +358,7 @@ class InputTile(base.InputTile, RasterInput):
358
358
  matching_method="gdal",
359
359
  matching_precision=8,
360
360
  matching_max_zoom=None,
361
+ **_,
361
362
  ):
362
363
  """
363
364
  Check if there is data within this tile.
@@ -328,7 +328,7 @@ class InputTile(base.InputTile, VectorInput):
328
328
  target_geometry_type=target_geometry_type,
329
329
  )
330
330
 
331
- def is_empty(self) -> bool:
331
+ def is_empty(self, **_) -> bool:
332
332
  """
333
333
  Check if there is data within this tile.
334
334
 
@@ -29,7 +29,7 @@ class InputTileProtocol(GridProtocol): # pragma: no cover
29
29
  """Read from input."""
30
30
  ...
31
31
 
32
- def is_empty(self) -> bool:
32
+ def is_empty(self, **kwargs) -> bool:
33
33
  """Checks if input is empty here."""
34
34
  ...
35
35
 
@@ -269,7 +269,7 @@ class ReferencedRasterInput(RasterInput):
269
269
  """Either read full array or resampled to grid."""
270
270
  return self.referenced_raster.read(indexes=indexes, resampling=resampling)
271
271
 
272
- def is_empty(self) -> bool:
272
+ def is_empty(self, **_) -> bool:
273
273
  """Checks if input is empty here."""
274
274
  return self.referenced_raster.masked_array().mask.all()
275
275