mapchete 0.24__tar.gz → 0.26__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (126) hide show
  1. home/ungarj/virtualenvs/p3/bin/mapchete +12 -0
  2. {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/__init__.py +1 -1
  3. home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/__pycache__/__init__.cpython-35.pyc +0 -0
  4. home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/__pycache__/_core.cpython-35.pyc +0 -0
  5. home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/__pycache__/config.cpython-35.pyc +0 -0
  6. home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/__pycache__/errors.cpython-35.pyc +0 -0
  7. home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/__pycache__/index.cpython-35.pyc +0 -0
  8. home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/__pycache__/log.cpython-35.pyc +0 -0
  9. home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/__pycache__/tile.cpython-35.pyc +0 -0
  10. {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/_core.py +45 -62
  11. home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/cli/__pycache__/__init__.cpython-35.pyc +0 -0
  12. home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/cli/__pycache__/create.cpython-35.pyc +0 -0
  13. home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/cli/__pycache__/execute.cpython-35.pyc +0 -0
  14. home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/cli/__pycache__/formats.cpython-35.pyc +0 -0
  15. home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/cli/__pycache__/index.cpython-35.pyc +0 -0
  16. home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/cli/__pycache__/main.cpython-35.pyc +0 -0
  17. home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/cli/__pycache__/pyramid.cpython-35.pyc +0 -0
  18. home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/cli/__pycache__/serve.cpython-35.pyc +0 -0
  19. home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/cli/__pycache__/tilify.cpython-35.pyc +0 -0
  20. home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/cli/__pycache__/utils.cpython-35.pyc +0 -0
  21. home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/cli/create.py +88 -0
  22. home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/cli/default/__pycache__/__init__.cpython-35.pyc +0 -0
  23. home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/cli/default/__pycache__/create.cpython-35.pyc +0 -0
  24. home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/cli/default/__pycache__/execute.cpython-35.pyc +0 -0
  25. home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/cli/default/__pycache__/formats.cpython-35.pyc +0 -0
  26. home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/cli/default/__pycache__/index.cpython-35.pyc +0 -0
  27. home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/cli/default/__pycache__/processes.cpython-35.pyc +0 -0
  28. home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/cli/default/__pycache__/pyramid.cpython-35.pyc +0 -0
  29. home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/cli/default/__pycache__/serve.cpython-35.pyc +0 -0
  30. home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/cli/default/execute.py +171 -0
  31. home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/cli/default/index.py +143 -0
  32. {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/cli/default/pyramid.py +0 -1
  33. {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/cli/default/serve.py +5 -4
  34. home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/cli/execute.py +115 -0
  35. home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/cli/formats.py +21 -0
  36. home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/cli/index.py +97 -0
  37. {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/cli/main.py +2 -3
  38. home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/cli/pyramid.py +127 -0
  39. home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/cli/serve.py +131 -0
  40. home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/cli/tilify.py +46 -0
  41. {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/cli/utils.py +6 -2
  42. home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/commons/__pycache__/__init__.cpython-35.pyc +0 -0
  43. home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/commons/__pycache__/clip.cpython-35.pyc +0 -0
  44. home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/commons/__pycache__/contours.cpython-35.pyc +0 -0
  45. home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/commons/__pycache__/hillshade.cpython-35.pyc +0 -0
  46. {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/commons/contours.py +1 -5
  47. {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/config.py +14 -22
  48. home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/formats/__pycache__/__init__.cpython-35.pyc +0 -0
  49. home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/formats/__pycache__/base.cpython-35.pyc +0 -0
  50. home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/formats/__pycache__/drivers.cpython-35.pyc +0 -0
  51. home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/formats/default/__pycache__/__init__.cpython-35.pyc +0 -0
  52. home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/formats/default/__pycache__/geojson.cpython-35.pyc +0 -0
  53. home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/formats/default/__pycache__/gtiff.cpython-35.pyc +0 -0
  54. home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/formats/default/__pycache__/mapchete_input.cpython-35.pyc +0 -0
  55. home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/formats/default/__pycache__/png.cpython-35.pyc +0 -0
  56. home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/formats/default/__pycache__/png_hillshade.cpython-35.pyc +0 -0
  57. home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/formats/default/__pycache__/raster_file.cpython-35.pyc +0 -0
  58. home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/formats/default/__pycache__/tile_directory.cpython-35.pyc +0 -0
  59. home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/formats/default/__pycache__/vector_file.cpython-35.pyc +0 -0
  60. {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/formats/default/geojson.py +20 -31
  61. {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/formats/default/gtiff.py +4 -10
  62. {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/formats/default/png.py +3 -9
  63. {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/formats/default/png_hillshade.py +3 -9
  64. {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/formats/default/tile_directory.py +5 -6
  65. home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/index.py +503 -0
  66. {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/io/__init__.py +68 -12
  67. home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/io/__pycache__/__init__.cpython-35.pyc +0 -0
  68. home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/io/__pycache__/raster.cpython-35.pyc +0 -0
  69. home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/io/__pycache__/vector.cpython-35.pyc +0 -0
  70. {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/io/raster.py +83 -33
  71. {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/io/vector.py +17 -14
  72. home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/static/__pycache__/__init__.cpython-35.pyc +0 -0
  73. home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/static/__pycache__/process_template.cpython-35.pyc +0 -0
  74. {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/tile.py +58 -16
  75. {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete-0.26-py3.5.egg-info}/PKG-INFO +27 -7
  76. {mapchete-0.24/mapchete.egg-info → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete-0.26-py3.5.egg-info}/SOURCES.txt +1 -1
  77. {mapchete-0.24/mapchete.egg-info → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete-0.26-py3.5.egg-info}/entry_points.txt +2 -1
  78. {mapchete-0.24/mapchete.egg-info → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete-0.26-py3.5.egg-info}/requires.txt +12 -3
  79. mapchete-0.24/README.rst +0 -165
  80. mapchete-0.24/mapchete/cli/default/execute.py +0 -123
  81. mapchete-0.24/mapchete/cli/default/index.py +0 -133
  82. mapchete-0.24/mapchete/index.py +0 -286
  83. mapchete-0.24/mapchete.egg-info/PKG-INFO +0 -182
  84. mapchete-0.24/setup.cfg +0 -10
  85. mapchete-0.24/setup.py +0 -93
  86. mapchete-0.24/test/test_cli.py +0 -529
  87. mapchete-0.24/test/test_commons.py +0 -85
  88. mapchete-0.24/test/test_config.py +0 -245
  89. mapchete-0.24/test/test_deprecated.py +0 -31
  90. mapchete-0.24/test/test_errors.py +0 -264
  91. mapchete-0.24/test/test_formats.py +0 -150
  92. mapchete-0.24/test/test_formats_geojson.py +0 -120
  93. mapchete-0.24/test/test_formats_geotiff.py +0 -175
  94. mapchete-0.24/test/test_formats_png.py +0 -107
  95. mapchete-0.24/test/test_formats_png_hillshade.py +0 -124
  96. mapchete-0.24/test/test_formats_tiledir_input.py +0 -148
  97. mapchete-0.24/test/test_io.py +0 -686
  98. mapchete-0.24/test/test_log.py +0 -17
  99. mapchete-0.24/test/test_mapchete.py +0 -417
  100. mapchete-0.24/test/test_processes.py +0 -85
  101. mapchete-0.24/test/test_timer.py +0 -24
  102. {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/cli/__init__.py +0 -0
  103. {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/cli/default/__init__.py +0 -0
  104. {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/cli/default/create.py +0 -0
  105. {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/cli/default/formats.py +0 -0
  106. {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/cli/default/processes.py +0 -0
  107. {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/commons/__init__.py +0 -0
  108. {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/commons/clip.py +0 -0
  109. {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/commons/hillshade.py +0 -0
  110. {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/errors.py +0 -0
  111. {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/formats/__init__.py +0 -0
  112. {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/formats/base.py +0 -0
  113. {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/formats/default/__init__.py +0 -0
  114. {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/formats/default/mapchete_input.py +0 -0
  115. {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/formats/default/raster_file.py +0 -0
  116. {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/formats/default/vector_file.py +0 -0
  117. {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/formats/drivers.py +0 -0
  118. {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/log.py +0 -0
  119. {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/static/__init__.py +0 -0
  120. {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/static/__init__.pyc +0 -0
  121. {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/static/index.html +0 -0
  122. {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/static/mapchete_template.mapchete +0 -0
  123. {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/static/process_template.py +0 -0
  124. {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/static/process_template.pyc +0 -0
  125. {mapchete-0.24/mapchete.egg-info → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete-0.26-py3.5.egg-info}/dependency_links.txt +0 -0
  126. {mapchete-0.24/mapchete.egg-info → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete-0.26-py3.5.egg-info}/top_level.txt +0 -0
@@ -0,0 +1,12 @@
1
+ #!/home/ungarj/virtualenvs/p3/bin/python
2
+ # EASY-INSTALL-ENTRY-SCRIPT: 'mapchete==0.26','console_scripts','mapchete'
3
+ __requires__ = 'mapchete==0.26'
4
+ import re
5
+ import sys
6
+ from pkg_resources import load_entry_point
7
+
8
+ if __name__ == '__main__':
9
+ sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0])
10
+ sys.exit(
11
+ load_entry_point('mapchete==0.26', 'console_scripts', 'mapchete')()
12
+ )
@@ -14,4 +14,4 @@ logging.getLogger("rasterio").setLevel(logging.ERROR)
14
14
  logging.getLogger("fiona").setLevel(logging.ERROR)
15
15
 
16
16
 
17
- __version__ = "0.24"
17
+ __version__ = "0.26"
@@ -2,17 +2,14 @@
2
2
 
3
3
  from cachetools import LRUCache
4
4
  from collections import namedtuple
5
- from functools import partial
5
+ import concurrent.futures
6
6
  import inspect
7
7
  from itertools import chain, product
8
8
  import logging
9
9
  from multiprocessing import cpu_count, current_process
10
- from multiprocessing.pool import Pool
11
10
  import numpy as np
12
11
  import numpy.ma as ma
13
12
  from shapely.geometry import shape
14
- import signal
15
- import six
16
13
  import threading
17
14
  from tilematrix import TilePyramid
18
15
  import time
@@ -197,13 +194,13 @@ class Mapchete(object):
197
194
  # run single tile
198
195
  if tile:
199
196
  yield _run_on_single_tile(self, tile)
200
- # run using multiprocessing
197
+ # run concurrently
201
198
  elif multi > 1:
202
199
  for process_info in _run_with_multiprocessing(
203
200
  self, list(_get_zoom_level(zoom, self)), multi, max_chunksize
204
201
  ):
205
202
  yield process_info
206
- # run without multiprocessing
203
+ # run sequentially
207
204
  elif multi == 1:
208
205
  for process_info in _run_without_multiprocessing(
209
206
  self, list(_get_zoom_level(zoom, self))
@@ -426,11 +423,11 @@ class Mapchete(object):
426
423
 
427
424
  def _read_existing_output(self, tile, output_tiles):
428
425
  if self.config.output.METADATA["data_type"] == "raster":
429
- mosaic, affine = raster.create_mosaic([
426
+ mosaic = raster.create_mosaic([
430
427
  (output_tile, self.read(output_tile))
431
428
  for output_tile in output_tiles
432
429
  ])
433
- return raster.extract_from_array(mosaic, affine, tile)
430
+ return raster.extract_from_array(mosaic.data, mosaic.affine, tile)
434
431
  elif self.config.output.METADATA["data_type"] == "vector":
435
432
  return list(chain.from_iterable([
436
433
  self.read(output_tile) for output_tile in output_tiles
@@ -500,19 +497,14 @@ class Mapchete(object):
500
497
  try:
501
498
  with Timer() as t:
502
499
  # Actually run process.
503
- if len(inspect.getargspec(self.config.process_func).args) == 1:
504
- process_data = self.config.process_func(tile_process)
505
- else:
506
- process_data = self.config.process_func(
507
- tile_process,
508
- **{
509
- k: v for k, v in six.iteritems(params)
510
- if k not in [
511
- "input", "output", "pyramid", "zoom_levels",
512
- "mapchete_file", "init_bounds", "init_zoom_levels"
513
- ]
514
- }
515
- )
500
+ process_data = self.config.process_func(
501
+ tile_process,
502
+ # only pass on kwargs which are defined in execute()
503
+ **{
504
+ k: v for k, v in params.items()
505
+ if k in inspect.signature(self.config.process_func).parameters
506
+ }
507
+ )
516
508
  except Exception as e:
517
509
  # Log process time
518
510
  logger.exception(
@@ -533,7 +525,7 @@ class Mapchete(object):
533
525
  return self.config.output.empty(process_tile)
534
526
 
535
527
  def _streamline_output(self, process_data):
536
- if isinstance(process_data, six.string_types) and (
528
+ if isinstance(process_data, str) and (
537
529
  process_data == "empty"
538
530
  ):
539
531
  raise MapcheteNodataTile
@@ -569,7 +561,7 @@ class Mapchete(object):
569
561
  )
570
562
  # resample from children tiles
571
563
  elif baselevel == "lower":
572
- mosaic, mosaic_affine = raster.create_mosaic([
564
+ mosaic = raster.create_mosaic([
573
565
  (
574
566
  child_tile,
575
567
  self.get_raw_output(child_tile, _baselevel_readonly=True)
@@ -579,8 +571,8 @@ class Mapchete(object):
579
571
  ).get_children()
580
572
  ])
581
573
  process_data = raster.resample_from_array(
582
- in_raster=mosaic,
583
- in_affine=mosaic_affine,
574
+ in_raster=mosaic.data,
575
+ in_affine=mosaic.affine,
584
576
  out_tile=tile,
585
577
  resampling=self.config.baselevels["lower"],
586
578
  nodataval=self.config.output.nodata
@@ -698,7 +690,7 @@ class MapcheteProcess(object):
698
690
  tiled input data : InputTile
699
691
  reprojected input data within tile
700
692
  """
701
- if not isinstance(input_id, six.string_types):
693
+ if not isinstance(input_id, str):
702
694
  return input_id.open(self.tile, **kwargs)
703
695
  if input_id not in self.params["input"]:
704
696
  raise ValueError("%s not found in config as input file" % input_id)
@@ -888,26 +880,35 @@ def _count_tiles(tiles, geometry, minzoom, maxzoom):
888
880
  for tile in tiles:
889
881
  # determine data covered by tile
890
882
  tile_intersection = tile.bbox().intersection(geometry)
883
+
891
884
  # skip if there is no data
892
885
  if tile_intersection.is_empty:
893
886
  continue
894
887
  # increase counter as tile contains data
895
888
  elif tile.zoom >= minzoom:
896
889
  count += 1
890
+
897
891
  # if there are further zoom levels, analyze descendants
898
892
  if tile.zoom < maxzoom:
899
- # if tile is full, all of its descendants will be full as well
900
- if tile.zoom >= minzoom and not tile_intersection.area < tile.bbox().area:
901
- # sum up tiles for each remaining zoom level
902
- count += sum([
903
- 4**z_diff
904
- for z_diff in range(1, (maxzoom - tile.zoom) + 1)
905
- ])
893
+
906
894
  # if tile is half full, analyze each descendant
907
- else:
895
+ if tile_intersection.area < tile.bbox().area:
908
896
  count += _count_tiles(
909
897
  tile.get_children(), tile_intersection, minzoom, maxzoom
910
898
  )
899
+
900
+ # if tile is full, all of its descendants will be full as well
901
+ else:
902
+ # sum up tiles for each remaining zoom level
903
+ count += sum([
904
+ 4**z for z in range(
905
+ # only count zoom levels which are greater than minzoom or
906
+ # count all zoom levels from tile zoom level to maxzoom
907
+ minzoom - tile.zoom if tile.zoom < minzoom else 1,
908
+ maxzoom - tile.zoom + 1
909
+ )
910
+ ])
911
+
911
912
  return count
912
913
 
913
914
 
@@ -922,39 +923,26 @@ def _run_on_single_tile(process, tile):
922
923
 
923
924
 
924
925
  def _run_with_multiprocessing(process, zoom_levels, multi, max_chunksize):
925
- logger.debug("run with multiprocessing")
926
+ logger.debug("run concurrently")
926
927
  num_processed = 0
927
928
  total_tiles = process.count_tiles(min(zoom_levels), max(zoom_levels))
928
929
  logger.debug("run process on %s tiles using %s workers", total_tiles, multi)
929
930
  with Timer() as t:
930
- f = partial(_process_worker, process)
931
- for zoom in zoom_levels:
932
- pool = Pool(multi, _worker_sigint_handler)
933
- try:
934
- for process_info in pool.imap_unordered(
935
- f,
936
- process.get_process_tiles(zoom),
937
- # set chunksize to between 1 and max_chunksize
938
- chunksize=max_chunksize
939
- ):
931
+ logger.debug("run process on %s tiles using %s workers", total_tiles, multi)
932
+ with concurrent.futures.ProcessPoolExecutor(max_workers=multi) as executor:
933
+ for zoom in zoom_levels:
934
+ for task in concurrent.futures.as_completed((
935
+ executor.submit(_process_worker, process, process_tile)
936
+ for process_tile in process.get_process_tiles(zoom)
937
+ )):
940
938
  num_processed += 1
941
939
  logger.debug("tile %s/%s finished", num_processed, total_tiles)
942
- yield process_info
943
- except KeyboardInterrupt:
944
- logger.error("Caught KeyboardInterrupt, terminating workers")
945
- pool.terminate()
946
- raise
947
- except Exception:
948
- pool.terminate()
949
- raise
950
- finally:
951
- pool.close()
952
- pool.join()
940
+ yield task.result()
953
941
  logger.debug("%s tile(s) iterated in %s", str(num_processed), t)
954
942
 
955
943
 
956
944
  def _run_without_multiprocessing(process, zoom_levels):
957
- logger.debug("run without multiprocessing")
945
+ logger.debug("run sequentially")
958
946
  num_processed = 0
959
947
  total_tiles = process.count_tiles(min(zoom_levels), max(zoom_levels))
960
948
  logger.debug("run process on %s tiles using 1 worker", total_tiles)
@@ -1015,8 +1003,3 @@ def _process_worker(process, process_tile):
1015
1003
  written=writer_info.written,
1016
1004
  write_msg=writer_info.write_msg
1017
1005
  )
1018
-
1019
-
1020
- def _worker_sigint_handler():
1021
- # ignore SIGINT and let everything be handled by parent process
1022
- signal.signal(signal.SIGINT, signal.SIG_IGN)
@@ -0,0 +1,88 @@
1
+ """Create dummy Mapchete and python process files."""
2
+
3
+ import os
4
+ from string import Template
5
+ from shutil import copyfile
6
+ from yaml import dump
7
+ import pkg_resources
8
+
9
+ FORMAT_MANDATORY = {
10
+ "GTiff": {
11
+ "bands": None,
12
+ "dtype": None
13
+ },
14
+ "PNG": {
15
+ "bands": None,
16
+ "dtype": None
17
+ },
18
+ "PNG_hillshade": {
19
+ "bands": 4,
20
+ "dtype": "uint8"
21
+ },
22
+ "GeoJSON": {
23
+ "schema": {}
24
+ },
25
+ "PostGIS": {
26
+ "schema": {},
27
+ "db_params": {
28
+ "host": None,
29
+ "port": None,
30
+ "db": None,
31
+ "user": None,
32
+ "password": None,
33
+ "table": None
34
+ }
35
+ }
36
+ }
37
+
38
+
39
+ def create_empty_process(args):
40
+ """
41
+ Create an empty Mapchete and process file in a given directory.
42
+
43
+ ``Namespace`` object should have the following attributes:
44
+ * process_file : name of the python file
45
+ * mapchete_file : name of the mapchete process configuration file
46
+ * out_format : optional output format
47
+ * out_path : optional output path
48
+ * pyramid_type : optional output type
49
+ * force : if True, it will replace already existing files
50
+
51
+ Parameters
52
+ ----------
53
+ args : argparse.Namespace
54
+ """
55
+ if os.path.isfile(args.process_file) or os.path.isfile(args.mapchete_file):
56
+ if not args.force:
57
+ raise IOError("file(s) already exists")
58
+
59
+ out_path = args.out_path if args.out_path else os.path.join(
60
+ os.getcwd(), "output")
61
+
62
+ # copy process file template to target directory
63
+ process_template = pkg_resources.resource_filename(
64
+ "mapchete.static", "process_template.py")
65
+ process_file = os.path.join(os.getcwd(), args.process_file)
66
+ copyfile(process_template, process_file)
67
+
68
+ # modify and copy mapchete file template to target directory
69
+ mapchete_template = pkg_resources.resource_filename(
70
+ "mapchete.static", "mapchete_template.mapchete")
71
+
72
+ output_options = {
73
+ 'format': args.out_format,
74
+ 'path': out_path}
75
+ output_options.update(FORMAT_MANDATORY[args.out_format])
76
+
77
+ pyramid_options = {'grid': args.pyramid_type}
78
+
79
+ substitute_elements = {
80
+ 'process_file': process_file,
81
+ 'output': dump({'output': output_options}, default_flow_style=False),
82
+ 'pyramid': dump({'pyramid': pyramid_options}, default_flow_style=False)
83
+ }
84
+ with open(mapchete_template, 'r') as config_template:
85
+ config = Template(config_template.read())
86
+ customized_config = config.substitute(substitute_elements)
87
+ with open(args.mapchete_file, 'w') as target_config:
88
+ target_config.write(customized_config)
@@ -0,0 +1,171 @@
1
+ """Command line utility to execute a Mapchete process."""
2
+
3
+ import click
4
+ import click_spinner
5
+ import logging
6
+ from multiprocessing import cpu_count
7
+ import os
8
+ from shapely import wkt
9
+ import sys
10
+ import tqdm
11
+ import yaml
12
+
13
+ import mapchete
14
+ from mapchete.cli import utils
15
+ from mapchete.config import get_zoom_levels, _map_to_new_config
16
+ from mapchete.index import zoom_index_gen
17
+ from mapchete.tile import BufferedTilePyramid
18
+
19
+
20
+ # workaround for https://github.com/tqdm/tqdm/issues/481
21
+ tqdm.monitor_interval = 0
22
+
23
+ logger = logging.getLogger(__name__)
24
+
25
+
26
+ @click.command(help="Execute a process.")
27
+ @utils.arg_mapchete_files
28
+ @utils.opt_zoom
29
+ @utils.opt_bounds
30
+ @utils.opt_point
31
+ @utils.opt_wkt_geometry
32
+ @utils.opt_tile
33
+ @utils.opt_overwrite
34
+ @utils.opt_multi
35
+ @utils.opt_input_file
36
+ @utils.opt_logfile
37
+ @utils.opt_verbose
38
+ @utils.opt_no_pbar
39
+ @utils.opt_debug
40
+ @utils.opt_max_chunksize
41
+ @utils.opt_vrt
42
+ @utils.opt_idx_out_dir
43
+ def execute(
44
+ mapchete_files,
45
+ zoom=None,
46
+ bounds=None,
47
+ point=None,
48
+ wkt_geometry=None,
49
+ tile=None,
50
+ overwrite=False,
51
+ multi=None,
52
+ input_file=None,
53
+ logfile=None,
54
+ verbose=False,
55
+ no_pbar=False,
56
+ debug=False,
57
+ max_chunksize=None,
58
+ vrt=False,
59
+ idx_out_dir=None
60
+ ):
61
+ """Execute a Mapchete process."""
62
+ multi = multi if multi else cpu_count()
63
+ mode = "overwrite" if overwrite else "continue"
64
+ # send verbose output to /dev/null if not activated
65
+ if debug or not verbose:
66
+ verbose_dst = open(os.devnull, 'w')
67
+ else:
68
+ verbose_dst = sys.stdout
69
+
70
+ for mapchete_file in mapchete_files:
71
+
72
+ tqdm.tqdm.write("preparing to process %s" % mapchete_file, file=verbose_dst)
73
+
74
+ with click_spinner.spinner(disable=debug) as spinner:
75
+
76
+ def _raw_conf():
77
+ return _map_to_new_config(
78
+ yaml.load(open(mapchete_file, "r").read())
79
+ )
80
+
81
+ def _tp():
82
+ return BufferedTilePyramid(
83
+ _raw_conf()["pyramid"]["grid"],
84
+ metatiling=_raw_conf()["pyramid"].get("metatiling", 1),
85
+ pixelbuffer=_raw_conf()["pyramid"].get("pixelbuffer", 0)
86
+ )
87
+
88
+ # process single tile
89
+ if tile:
90
+ tile = _tp().tile(*tile)
91
+ with mapchete.open(
92
+ mapchete_file, mode=mode, bounds=tile.bounds,
93
+ zoom=tile.zoom, single_input_file=input_file
94
+ ) as mp:
95
+ spinner.stop()
96
+ tqdm.tqdm.write("processing 1 tile")
97
+ for result in mp.batch_processor(tile=tile):
98
+ utils.write_verbose_msg(result, dst=verbose_dst)
99
+
100
+ if vrt:
101
+ tqdm.tqdm.write("creating VRT")
102
+ for tile in tqdm.tqdm(
103
+ zoom_index_gen(
104
+ mp=mp,
105
+ zoom=tile.zoom,
106
+ out_dir=(
107
+ idx_out_dir if idx_out_dir else mp.config.output.path
108
+ ),
109
+ vrt=vrt,
110
+ ),
111
+ total=mp.count_tiles(tile.zoom, tile.zoom),
112
+ unit="tile",
113
+ disable=debug
114
+ ):
115
+ logger.debug(tile)
116
+
117
+ # initialize and run process
118
+ else:
119
+ if wkt_geometry:
120
+ bounds = wkt.loads(wkt_geometry).bounds
121
+ elif point:
122
+ x, y = point
123
+ zoom_levels = get_zoom_levels(
124
+ process_zoom_levels=_raw_conf()["zoom_levels"],
125
+ init_zoom_levels=zoom
126
+ )
127
+ bounds = _tp().tile_from_xy(x, y, max(zoom_levels)).bounds
128
+ else:
129
+ bounds = bounds
130
+ with mapchete.open(
131
+ mapchete_file, bounds=bounds, zoom=zoom,
132
+ mode=mode, single_input_file=input_file
133
+ ) as mp:
134
+ spinner.stop()
135
+ tiles_count = mp.count_tiles(
136
+ min(mp.config.init_zoom_levels),
137
+ max(mp.config.init_zoom_levels))
138
+ tqdm.tqdm.write(
139
+ "processing %s tile(s) on %s worker(s)" % (tiles_count, multi)
140
+ )
141
+ for process_info in tqdm.tqdm(
142
+ mp.batch_processor(
143
+ multi=multi, zoom=zoom,
144
+ max_chunksize=max_chunksize),
145
+ total=tiles_count,
146
+ unit="tile",
147
+ disable=debug or no_pbar
148
+ ):
149
+ utils.write_verbose_msg(process_info, dst=verbose_dst)
150
+
151
+ if vrt:
152
+ tqdm.tqdm.write("creating VRT(s)")
153
+ for z in mp.config.init_zoom_levels:
154
+ logger.debug("zoom %s", z)
155
+ for tile in tqdm.tqdm(
156
+ zoom_index_gen(
157
+ mp=mp,
158
+ zoom=z,
159
+ out_dir=(
160
+ idx_out_dir if idx_out_dir
161
+ else mp.config.output.path
162
+ ),
163
+ vrt=vrt
164
+ ),
165
+ total=mp.count_tiles(z, z),
166
+ unit="tile",
167
+ disable=debug
168
+ ):
169
+ logger.debug(tile)
170
+
171
+ tqdm.tqdm.write("process finished", file=verbose_dst)