mapchete 0.24__tar.gz → 0.26__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- home/ungarj/virtualenvs/p3/bin/mapchete +12 -0
- {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/__init__.py +1 -1
- home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/__pycache__/__init__.cpython-35.pyc +0 -0
- home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/__pycache__/_core.cpython-35.pyc +0 -0
- home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/__pycache__/config.cpython-35.pyc +0 -0
- home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/__pycache__/errors.cpython-35.pyc +0 -0
- home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/__pycache__/index.cpython-35.pyc +0 -0
- home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/__pycache__/log.cpython-35.pyc +0 -0
- home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/__pycache__/tile.cpython-35.pyc +0 -0
- {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/_core.py +45 -62
- home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/cli/__pycache__/__init__.cpython-35.pyc +0 -0
- home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/cli/__pycache__/create.cpython-35.pyc +0 -0
- home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/cli/__pycache__/execute.cpython-35.pyc +0 -0
- home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/cli/__pycache__/formats.cpython-35.pyc +0 -0
- home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/cli/__pycache__/index.cpython-35.pyc +0 -0
- home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/cli/__pycache__/main.cpython-35.pyc +0 -0
- home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/cli/__pycache__/pyramid.cpython-35.pyc +0 -0
- home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/cli/__pycache__/serve.cpython-35.pyc +0 -0
- home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/cli/__pycache__/tilify.cpython-35.pyc +0 -0
- home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/cli/__pycache__/utils.cpython-35.pyc +0 -0
- home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/cli/create.py +88 -0
- home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/cli/default/__pycache__/__init__.cpython-35.pyc +0 -0
- home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/cli/default/__pycache__/create.cpython-35.pyc +0 -0
- home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/cli/default/__pycache__/execute.cpython-35.pyc +0 -0
- home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/cli/default/__pycache__/formats.cpython-35.pyc +0 -0
- home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/cli/default/__pycache__/index.cpython-35.pyc +0 -0
- home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/cli/default/__pycache__/processes.cpython-35.pyc +0 -0
- home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/cli/default/__pycache__/pyramid.cpython-35.pyc +0 -0
- home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/cli/default/__pycache__/serve.cpython-35.pyc +0 -0
- home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/cli/default/execute.py +171 -0
- home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/cli/default/index.py +143 -0
- {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/cli/default/pyramid.py +0 -1
- {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/cli/default/serve.py +5 -4
- home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/cli/execute.py +115 -0
- home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/cli/formats.py +21 -0
- home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/cli/index.py +97 -0
- {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/cli/main.py +2 -3
- home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/cli/pyramid.py +127 -0
- home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/cli/serve.py +131 -0
- home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/cli/tilify.py +46 -0
- {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/cli/utils.py +6 -2
- home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/commons/__pycache__/__init__.cpython-35.pyc +0 -0
- home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/commons/__pycache__/clip.cpython-35.pyc +0 -0
- home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/commons/__pycache__/contours.cpython-35.pyc +0 -0
- home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/commons/__pycache__/hillshade.cpython-35.pyc +0 -0
- {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/commons/contours.py +1 -5
- {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/config.py +14 -22
- home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/formats/__pycache__/__init__.cpython-35.pyc +0 -0
- home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/formats/__pycache__/base.cpython-35.pyc +0 -0
- home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/formats/__pycache__/drivers.cpython-35.pyc +0 -0
- home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/formats/default/__pycache__/__init__.cpython-35.pyc +0 -0
- home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/formats/default/__pycache__/geojson.cpython-35.pyc +0 -0
- home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/formats/default/__pycache__/gtiff.cpython-35.pyc +0 -0
- home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/formats/default/__pycache__/mapchete_input.cpython-35.pyc +0 -0
- home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/formats/default/__pycache__/png.cpython-35.pyc +0 -0
- home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/formats/default/__pycache__/png_hillshade.cpython-35.pyc +0 -0
- home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/formats/default/__pycache__/raster_file.cpython-35.pyc +0 -0
- home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/formats/default/__pycache__/tile_directory.cpython-35.pyc +0 -0
- home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/formats/default/__pycache__/vector_file.cpython-35.pyc +0 -0
- {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/formats/default/geojson.py +20 -31
- {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/formats/default/gtiff.py +4 -10
- {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/formats/default/png.py +3 -9
- {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/formats/default/png_hillshade.py +3 -9
- {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/formats/default/tile_directory.py +5 -6
- home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/index.py +503 -0
- {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/io/__init__.py +68 -12
- home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/io/__pycache__/__init__.cpython-35.pyc +0 -0
- home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/io/__pycache__/raster.cpython-35.pyc +0 -0
- home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/io/__pycache__/vector.cpython-35.pyc +0 -0
- {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/io/raster.py +83 -33
- {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/io/vector.py +17 -14
- home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/static/__pycache__/__init__.cpython-35.pyc +0 -0
- home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/static/__pycache__/process_template.cpython-35.pyc +0 -0
- {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/tile.py +58 -16
- {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete-0.26-py3.5.egg-info}/PKG-INFO +27 -7
- {mapchete-0.24/mapchete.egg-info → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete-0.26-py3.5.egg-info}/SOURCES.txt +1 -1
- {mapchete-0.24/mapchete.egg-info → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete-0.26-py3.5.egg-info}/entry_points.txt +2 -1
- {mapchete-0.24/mapchete.egg-info → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete-0.26-py3.5.egg-info}/requires.txt +12 -3
- mapchete-0.24/README.rst +0 -165
- mapchete-0.24/mapchete/cli/default/execute.py +0 -123
- mapchete-0.24/mapchete/cli/default/index.py +0 -133
- mapchete-0.24/mapchete/index.py +0 -286
- mapchete-0.24/mapchete.egg-info/PKG-INFO +0 -182
- mapchete-0.24/setup.cfg +0 -10
- mapchete-0.24/setup.py +0 -93
- mapchete-0.24/test/test_cli.py +0 -529
- mapchete-0.24/test/test_commons.py +0 -85
- mapchete-0.24/test/test_config.py +0 -245
- mapchete-0.24/test/test_deprecated.py +0 -31
- mapchete-0.24/test/test_errors.py +0 -264
- mapchete-0.24/test/test_formats.py +0 -150
- mapchete-0.24/test/test_formats_geojson.py +0 -120
- mapchete-0.24/test/test_formats_geotiff.py +0 -175
- mapchete-0.24/test/test_formats_png.py +0 -107
- mapchete-0.24/test/test_formats_png_hillshade.py +0 -124
- mapchete-0.24/test/test_formats_tiledir_input.py +0 -148
- mapchete-0.24/test/test_io.py +0 -686
- mapchete-0.24/test/test_log.py +0 -17
- mapchete-0.24/test/test_mapchete.py +0 -417
- mapchete-0.24/test/test_processes.py +0 -85
- mapchete-0.24/test/test_timer.py +0 -24
- {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/cli/__init__.py +0 -0
- {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/cli/default/__init__.py +0 -0
- {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/cli/default/create.py +0 -0
- {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/cli/default/formats.py +0 -0
- {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/cli/default/processes.py +0 -0
- {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/commons/__init__.py +0 -0
- {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/commons/clip.py +0 -0
- {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/commons/hillshade.py +0 -0
- {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/errors.py +0 -0
- {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/formats/__init__.py +0 -0
- {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/formats/base.py +0 -0
- {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/formats/default/__init__.py +0 -0
- {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/formats/default/mapchete_input.py +0 -0
- {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/formats/default/raster_file.py +0 -0
- {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/formats/default/vector_file.py +0 -0
- {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/formats/drivers.py +0 -0
- {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/log.py +0 -0
- {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/static/__init__.py +0 -0
- {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/static/__init__.pyc +0 -0
- {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/static/index.html +0 -0
- {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/static/mapchete_template.mapchete +0 -0
- {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/static/process_template.py +0 -0
- {mapchete-0.24 → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages}/mapchete/static/process_template.pyc +0 -0
- {mapchete-0.24/mapchete.egg-info → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete-0.26-py3.5.egg-info}/dependency_links.txt +0 -0
- {mapchete-0.24/mapchete.egg-info → home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete-0.26-py3.5.egg-info}/top_level.txt +0 -0
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
#!/home/ungarj/virtualenvs/p3/bin/python
|
|
2
|
+
# EASY-INSTALL-ENTRY-SCRIPT: 'mapchete==0.26','console_scripts','mapchete'
|
|
3
|
+
__requires__ = 'mapchete==0.26'
|
|
4
|
+
import re
|
|
5
|
+
import sys
|
|
6
|
+
from pkg_resources import load_entry_point
|
|
7
|
+
|
|
8
|
+
if __name__ == '__main__':
|
|
9
|
+
sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0])
|
|
10
|
+
sys.exit(
|
|
11
|
+
load_entry_point('mapchete==0.26', 'console_scripts', 'mapchete')()
|
|
12
|
+
)
|
home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/__pycache__/__init__.cpython-35.pyc
ADDED
|
Binary file
|
home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/__pycache__/_core.cpython-35.pyc
ADDED
|
Binary file
|
home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/__pycache__/config.cpython-35.pyc
ADDED
|
Binary file
|
home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/__pycache__/errors.cpython-35.pyc
ADDED
|
Binary file
|
home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/__pycache__/index.cpython-35.pyc
ADDED
|
Binary file
|
home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/__pycache__/log.cpython-35.pyc
ADDED
|
Binary file
|
home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/__pycache__/tile.cpython-35.pyc
ADDED
|
Binary file
|
|
@@ -2,17 +2,14 @@
|
|
|
2
2
|
|
|
3
3
|
from cachetools import LRUCache
|
|
4
4
|
from collections import namedtuple
|
|
5
|
-
|
|
5
|
+
import concurrent.futures
|
|
6
6
|
import inspect
|
|
7
7
|
from itertools import chain, product
|
|
8
8
|
import logging
|
|
9
9
|
from multiprocessing import cpu_count, current_process
|
|
10
|
-
from multiprocessing.pool import Pool
|
|
11
10
|
import numpy as np
|
|
12
11
|
import numpy.ma as ma
|
|
13
12
|
from shapely.geometry import shape
|
|
14
|
-
import signal
|
|
15
|
-
import six
|
|
16
13
|
import threading
|
|
17
14
|
from tilematrix import TilePyramid
|
|
18
15
|
import time
|
|
@@ -197,13 +194,13 @@ class Mapchete(object):
|
|
|
197
194
|
# run single tile
|
|
198
195
|
if tile:
|
|
199
196
|
yield _run_on_single_tile(self, tile)
|
|
200
|
-
# run
|
|
197
|
+
# run concurrently
|
|
201
198
|
elif multi > 1:
|
|
202
199
|
for process_info in _run_with_multiprocessing(
|
|
203
200
|
self, list(_get_zoom_level(zoom, self)), multi, max_chunksize
|
|
204
201
|
):
|
|
205
202
|
yield process_info
|
|
206
|
-
# run
|
|
203
|
+
# run sequentially
|
|
207
204
|
elif multi == 1:
|
|
208
205
|
for process_info in _run_without_multiprocessing(
|
|
209
206
|
self, list(_get_zoom_level(zoom, self))
|
|
@@ -426,11 +423,11 @@ class Mapchete(object):
|
|
|
426
423
|
|
|
427
424
|
def _read_existing_output(self, tile, output_tiles):
|
|
428
425
|
if self.config.output.METADATA["data_type"] == "raster":
|
|
429
|
-
mosaic
|
|
426
|
+
mosaic = raster.create_mosaic([
|
|
430
427
|
(output_tile, self.read(output_tile))
|
|
431
428
|
for output_tile in output_tiles
|
|
432
429
|
])
|
|
433
|
-
return raster.extract_from_array(mosaic, affine, tile)
|
|
430
|
+
return raster.extract_from_array(mosaic.data, mosaic.affine, tile)
|
|
434
431
|
elif self.config.output.METADATA["data_type"] == "vector":
|
|
435
432
|
return list(chain.from_iterable([
|
|
436
433
|
self.read(output_tile) for output_tile in output_tiles
|
|
@@ -500,19 +497,14 @@ class Mapchete(object):
|
|
|
500
497
|
try:
|
|
501
498
|
with Timer() as t:
|
|
502
499
|
# Actually run process.
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
"input", "output", "pyramid", "zoom_levels",
|
|
512
|
-
"mapchete_file", "init_bounds", "init_zoom_levels"
|
|
513
|
-
]
|
|
514
|
-
}
|
|
515
|
-
)
|
|
500
|
+
process_data = self.config.process_func(
|
|
501
|
+
tile_process,
|
|
502
|
+
# only pass on kwargs which are defined in execute()
|
|
503
|
+
**{
|
|
504
|
+
k: v for k, v in params.items()
|
|
505
|
+
if k in inspect.signature(self.config.process_func).parameters
|
|
506
|
+
}
|
|
507
|
+
)
|
|
516
508
|
except Exception as e:
|
|
517
509
|
# Log process time
|
|
518
510
|
logger.exception(
|
|
@@ -533,7 +525,7 @@ class Mapchete(object):
|
|
|
533
525
|
return self.config.output.empty(process_tile)
|
|
534
526
|
|
|
535
527
|
def _streamline_output(self, process_data):
|
|
536
|
-
if isinstance(process_data,
|
|
528
|
+
if isinstance(process_data, str) and (
|
|
537
529
|
process_data == "empty"
|
|
538
530
|
):
|
|
539
531
|
raise MapcheteNodataTile
|
|
@@ -569,7 +561,7 @@ class Mapchete(object):
|
|
|
569
561
|
)
|
|
570
562
|
# resample from children tiles
|
|
571
563
|
elif baselevel == "lower":
|
|
572
|
-
mosaic
|
|
564
|
+
mosaic = raster.create_mosaic([
|
|
573
565
|
(
|
|
574
566
|
child_tile,
|
|
575
567
|
self.get_raw_output(child_tile, _baselevel_readonly=True)
|
|
@@ -579,8 +571,8 @@ class Mapchete(object):
|
|
|
579
571
|
).get_children()
|
|
580
572
|
])
|
|
581
573
|
process_data = raster.resample_from_array(
|
|
582
|
-
in_raster=mosaic,
|
|
583
|
-
in_affine=
|
|
574
|
+
in_raster=mosaic.data,
|
|
575
|
+
in_affine=mosaic.affine,
|
|
584
576
|
out_tile=tile,
|
|
585
577
|
resampling=self.config.baselevels["lower"],
|
|
586
578
|
nodataval=self.config.output.nodata
|
|
@@ -698,7 +690,7 @@ class MapcheteProcess(object):
|
|
|
698
690
|
tiled input data : InputTile
|
|
699
691
|
reprojected input data within tile
|
|
700
692
|
"""
|
|
701
|
-
if not isinstance(input_id,
|
|
693
|
+
if not isinstance(input_id, str):
|
|
702
694
|
return input_id.open(self.tile, **kwargs)
|
|
703
695
|
if input_id not in self.params["input"]:
|
|
704
696
|
raise ValueError("%s not found in config as input file" % input_id)
|
|
@@ -888,26 +880,35 @@ def _count_tiles(tiles, geometry, minzoom, maxzoom):
|
|
|
888
880
|
for tile in tiles:
|
|
889
881
|
# determine data covered by tile
|
|
890
882
|
tile_intersection = tile.bbox().intersection(geometry)
|
|
883
|
+
|
|
891
884
|
# skip if there is no data
|
|
892
885
|
if tile_intersection.is_empty:
|
|
893
886
|
continue
|
|
894
887
|
# increase counter as tile contains data
|
|
895
888
|
elif tile.zoom >= minzoom:
|
|
896
889
|
count += 1
|
|
890
|
+
|
|
897
891
|
# if there are further zoom levels, analyze descendants
|
|
898
892
|
if tile.zoom < maxzoom:
|
|
899
|
-
|
|
900
|
-
if tile.zoom >= minzoom and not tile_intersection.area < tile.bbox().area:
|
|
901
|
-
# sum up tiles for each remaining zoom level
|
|
902
|
-
count += sum([
|
|
903
|
-
4**z_diff
|
|
904
|
-
for z_diff in range(1, (maxzoom - tile.zoom) + 1)
|
|
905
|
-
])
|
|
893
|
+
|
|
906
894
|
# if tile is half full, analyze each descendant
|
|
907
|
-
|
|
895
|
+
if tile_intersection.area < tile.bbox().area:
|
|
908
896
|
count += _count_tiles(
|
|
909
897
|
tile.get_children(), tile_intersection, minzoom, maxzoom
|
|
910
898
|
)
|
|
899
|
+
|
|
900
|
+
# if tile is full, all of its descendants will be full as well
|
|
901
|
+
else:
|
|
902
|
+
# sum up tiles for each remaining zoom level
|
|
903
|
+
count += sum([
|
|
904
|
+
4**z for z in range(
|
|
905
|
+
# only count zoom levels which are greater than minzoom or
|
|
906
|
+
# count all zoom levels from tile zoom level to maxzoom
|
|
907
|
+
minzoom - tile.zoom if tile.zoom < minzoom else 1,
|
|
908
|
+
maxzoom - tile.zoom + 1
|
|
909
|
+
)
|
|
910
|
+
])
|
|
911
|
+
|
|
911
912
|
return count
|
|
912
913
|
|
|
913
914
|
|
|
@@ -922,39 +923,26 @@ def _run_on_single_tile(process, tile):
|
|
|
922
923
|
|
|
923
924
|
|
|
924
925
|
def _run_with_multiprocessing(process, zoom_levels, multi, max_chunksize):
|
|
925
|
-
logger.debug("run
|
|
926
|
+
logger.debug("run concurrently")
|
|
926
927
|
num_processed = 0
|
|
927
928
|
total_tiles = process.count_tiles(min(zoom_levels), max(zoom_levels))
|
|
928
929
|
logger.debug("run process on %s tiles using %s workers", total_tiles, multi)
|
|
929
930
|
with Timer() as t:
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
# set chunksize to between 1 and max_chunksize
|
|
938
|
-
chunksize=max_chunksize
|
|
939
|
-
):
|
|
931
|
+
logger.debug("run process on %s tiles using %s workers", total_tiles, multi)
|
|
932
|
+
with concurrent.futures.ProcessPoolExecutor(max_workers=multi) as executor:
|
|
933
|
+
for zoom in zoom_levels:
|
|
934
|
+
for task in concurrent.futures.as_completed((
|
|
935
|
+
executor.submit(_process_worker, process, process_tile)
|
|
936
|
+
for process_tile in process.get_process_tiles(zoom)
|
|
937
|
+
)):
|
|
940
938
|
num_processed += 1
|
|
941
939
|
logger.debug("tile %s/%s finished", num_processed, total_tiles)
|
|
942
|
-
yield
|
|
943
|
-
except KeyboardInterrupt:
|
|
944
|
-
logger.error("Caught KeyboardInterrupt, terminating workers")
|
|
945
|
-
pool.terminate()
|
|
946
|
-
raise
|
|
947
|
-
except Exception:
|
|
948
|
-
pool.terminate()
|
|
949
|
-
raise
|
|
950
|
-
finally:
|
|
951
|
-
pool.close()
|
|
952
|
-
pool.join()
|
|
940
|
+
yield task.result()
|
|
953
941
|
logger.debug("%s tile(s) iterated in %s", str(num_processed), t)
|
|
954
942
|
|
|
955
943
|
|
|
956
944
|
def _run_without_multiprocessing(process, zoom_levels):
|
|
957
|
-
logger.debug("run
|
|
945
|
+
logger.debug("run sequentially")
|
|
958
946
|
num_processed = 0
|
|
959
947
|
total_tiles = process.count_tiles(min(zoom_levels), max(zoom_levels))
|
|
960
948
|
logger.debug("run process on %s tiles using 1 worker", total_tiles)
|
|
@@ -1015,8 +1003,3 @@ def _process_worker(process, process_tile):
|
|
|
1015
1003
|
written=writer_info.written,
|
|
1016
1004
|
write_msg=writer_info.write_msg
|
|
1017
1005
|
)
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
def _worker_sigint_handler():
|
|
1021
|
-
# ignore SIGINT and let everything be handled by parent process
|
|
1022
|
-
signal.signal(signal.SIGINT, signal.SIG_IGN)
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/cli/__pycache__/index.cpython-35.pyc
ADDED
|
Binary file
|
home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/cli/__pycache__/main.cpython-35.pyc
ADDED
|
Binary file
|
|
Binary file
|
home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/cli/__pycache__/serve.cpython-35.pyc
ADDED
|
Binary file
|
|
Binary file
|
home/ungarj/virtualenvs/p3/lib/python3.5/site-packages/mapchete/cli/__pycache__/utils.cpython-35.pyc
ADDED
|
Binary file
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
"""Create dummy Mapchete and python process files."""
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
from string import Template
|
|
5
|
+
from shutil import copyfile
|
|
6
|
+
from yaml import dump
|
|
7
|
+
import pkg_resources
|
|
8
|
+
|
|
9
|
+
FORMAT_MANDATORY = {
|
|
10
|
+
"GTiff": {
|
|
11
|
+
"bands": None,
|
|
12
|
+
"dtype": None
|
|
13
|
+
},
|
|
14
|
+
"PNG": {
|
|
15
|
+
"bands": None,
|
|
16
|
+
"dtype": None
|
|
17
|
+
},
|
|
18
|
+
"PNG_hillshade": {
|
|
19
|
+
"bands": 4,
|
|
20
|
+
"dtype": "uint8"
|
|
21
|
+
},
|
|
22
|
+
"GeoJSON": {
|
|
23
|
+
"schema": {}
|
|
24
|
+
},
|
|
25
|
+
"PostGIS": {
|
|
26
|
+
"schema": {},
|
|
27
|
+
"db_params": {
|
|
28
|
+
"host": None,
|
|
29
|
+
"port": None,
|
|
30
|
+
"db": None,
|
|
31
|
+
"user": None,
|
|
32
|
+
"password": None,
|
|
33
|
+
"table": None
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def create_empty_process(args):
|
|
40
|
+
"""
|
|
41
|
+
Create an empty Mapchete and process file in a given directory.
|
|
42
|
+
|
|
43
|
+
``Namespace`` object should have the following attributes:
|
|
44
|
+
* process_file : name of the python file
|
|
45
|
+
* mapchete_file : name of the mapchete process configuration file
|
|
46
|
+
* out_format : optional output format
|
|
47
|
+
* out_path : optional output path
|
|
48
|
+
* pyramid_type : optional output type
|
|
49
|
+
* force : if True, it will replace already existing files
|
|
50
|
+
|
|
51
|
+
Parameters
|
|
52
|
+
----------
|
|
53
|
+
args : argparse.Namespace
|
|
54
|
+
"""
|
|
55
|
+
if os.path.isfile(args.process_file) or os.path.isfile(args.mapchete_file):
|
|
56
|
+
if not args.force:
|
|
57
|
+
raise IOError("file(s) already exists")
|
|
58
|
+
|
|
59
|
+
out_path = args.out_path if args.out_path else os.path.join(
|
|
60
|
+
os.getcwd(), "output")
|
|
61
|
+
|
|
62
|
+
# copy process file template to target directory
|
|
63
|
+
process_template = pkg_resources.resource_filename(
|
|
64
|
+
"mapchete.static", "process_template.py")
|
|
65
|
+
process_file = os.path.join(os.getcwd(), args.process_file)
|
|
66
|
+
copyfile(process_template, process_file)
|
|
67
|
+
|
|
68
|
+
# modify and copy mapchete file template to target directory
|
|
69
|
+
mapchete_template = pkg_resources.resource_filename(
|
|
70
|
+
"mapchete.static", "mapchete_template.mapchete")
|
|
71
|
+
|
|
72
|
+
output_options = {
|
|
73
|
+
'format': args.out_format,
|
|
74
|
+
'path': out_path}
|
|
75
|
+
output_options.update(FORMAT_MANDATORY[args.out_format])
|
|
76
|
+
|
|
77
|
+
pyramid_options = {'grid': args.pyramid_type}
|
|
78
|
+
|
|
79
|
+
substitute_elements = {
|
|
80
|
+
'process_file': process_file,
|
|
81
|
+
'output': dump({'output': output_options}, default_flow_style=False),
|
|
82
|
+
'pyramid': dump({'pyramid': pyramid_options}, default_flow_style=False)
|
|
83
|
+
}
|
|
84
|
+
with open(mapchete_template, 'r') as config_template:
|
|
85
|
+
config = Template(config_template.read())
|
|
86
|
+
customized_config = config.substitute(substitute_elements)
|
|
87
|
+
with open(args.mapchete_file, 'w') as target_config:
|
|
88
|
+
target_config.write(customized_config)
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
"""Command line utility to execute a Mapchete process."""
|
|
2
|
+
|
|
3
|
+
import click
|
|
4
|
+
import click_spinner
|
|
5
|
+
import logging
|
|
6
|
+
from multiprocessing import cpu_count
|
|
7
|
+
import os
|
|
8
|
+
from shapely import wkt
|
|
9
|
+
import sys
|
|
10
|
+
import tqdm
|
|
11
|
+
import yaml
|
|
12
|
+
|
|
13
|
+
import mapchete
|
|
14
|
+
from mapchete.cli import utils
|
|
15
|
+
from mapchete.config import get_zoom_levels, _map_to_new_config
|
|
16
|
+
from mapchete.index import zoom_index_gen
|
|
17
|
+
from mapchete.tile import BufferedTilePyramid
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
# workaround for https://github.com/tqdm/tqdm/issues/481
|
|
21
|
+
tqdm.monitor_interval = 0
|
|
22
|
+
|
|
23
|
+
logger = logging.getLogger(__name__)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@click.command(help="Execute a process.")
|
|
27
|
+
@utils.arg_mapchete_files
|
|
28
|
+
@utils.opt_zoom
|
|
29
|
+
@utils.opt_bounds
|
|
30
|
+
@utils.opt_point
|
|
31
|
+
@utils.opt_wkt_geometry
|
|
32
|
+
@utils.opt_tile
|
|
33
|
+
@utils.opt_overwrite
|
|
34
|
+
@utils.opt_multi
|
|
35
|
+
@utils.opt_input_file
|
|
36
|
+
@utils.opt_logfile
|
|
37
|
+
@utils.opt_verbose
|
|
38
|
+
@utils.opt_no_pbar
|
|
39
|
+
@utils.opt_debug
|
|
40
|
+
@utils.opt_max_chunksize
|
|
41
|
+
@utils.opt_vrt
|
|
42
|
+
@utils.opt_idx_out_dir
|
|
43
|
+
def execute(
|
|
44
|
+
mapchete_files,
|
|
45
|
+
zoom=None,
|
|
46
|
+
bounds=None,
|
|
47
|
+
point=None,
|
|
48
|
+
wkt_geometry=None,
|
|
49
|
+
tile=None,
|
|
50
|
+
overwrite=False,
|
|
51
|
+
multi=None,
|
|
52
|
+
input_file=None,
|
|
53
|
+
logfile=None,
|
|
54
|
+
verbose=False,
|
|
55
|
+
no_pbar=False,
|
|
56
|
+
debug=False,
|
|
57
|
+
max_chunksize=None,
|
|
58
|
+
vrt=False,
|
|
59
|
+
idx_out_dir=None
|
|
60
|
+
):
|
|
61
|
+
"""Execute a Mapchete process."""
|
|
62
|
+
multi = multi if multi else cpu_count()
|
|
63
|
+
mode = "overwrite" if overwrite else "continue"
|
|
64
|
+
# send verbose output to /dev/null if not activated
|
|
65
|
+
if debug or not verbose:
|
|
66
|
+
verbose_dst = open(os.devnull, 'w')
|
|
67
|
+
else:
|
|
68
|
+
verbose_dst = sys.stdout
|
|
69
|
+
|
|
70
|
+
for mapchete_file in mapchete_files:
|
|
71
|
+
|
|
72
|
+
tqdm.tqdm.write("preparing to process %s" % mapchete_file, file=verbose_dst)
|
|
73
|
+
|
|
74
|
+
with click_spinner.spinner(disable=debug) as spinner:
|
|
75
|
+
|
|
76
|
+
def _raw_conf():
|
|
77
|
+
return _map_to_new_config(
|
|
78
|
+
yaml.load(open(mapchete_file, "r").read())
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
def _tp():
|
|
82
|
+
return BufferedTilePyramid(
|
|
83
|
+
_raw_conf()["pyramid"]["grid"],
|
|
84
|
+
metatiling=_raw_conf()["pyramid"].get("metatiling", 1),
|
|
85
|
+
pixelbuffer=_raw_conf()["pyramid"].get("pixelbuffer", 0)
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
# process single tile
|
|
89
|
+
if tile:
|
|
90
|
+
tile = _tp().tile(*tile)
|
|
91
|
+
with mapchete.open(
|
|
92
|
+
mapchete_file, mode=mode, bounds=tile.bounds,
|
|
93
|
+
zoom=tile.zoom, single_input_file=input_file
|
|
94
|
+
) as mp:
|
|
95
|
+
spinner.stop()
|
|
96
|
+
tqdm.tqdm.write("processing 1 tile")
|
|
97
|
+
for result in mp.batch_processor(tile=tile):
|
|
98
|
+
utils.write_verbose_msg(result, dst=verbose_dst)
|
|
99
|
+
|
|
100
|
+
if vrt:
|
|
101
|
+
tqdm.tqdm.write("creating VRT")
|
|
102
|
+
for tile in tqdm.tqdm(
|
|
103
|
+
zoom_index_gen(
|
|
104
|
+
mp=mp,
|
|
105
|
+
zoom=tile.zoom,
|
|
106
|
+
out_dir=(
|
|
107
|
+
idx_out_dir if idx_out_dir else mp.config.output.path
|
|
108
|
+
),
|
|
109
|
+
vrt=vrt,
|
|
110
|
+
),
|
|
111
|
+
total=mp.count_tiles(tile.zoom, tile.zoom),
|
|
112
|
+
unit="tile",
|
|
113
|
+
disable=debug
|
|
114
|
+
):
|
|
115
|
+
logger.debug(tile)
|
|
116
|
+
|
|
117
|
+
# initialize and run process
|
|
118
|
+
else:
|
|
119
|
+
if wkt_geometry:
|
|
120
|
+
bounds = wkt.loads(wkt_geometry).bounds
|
|
121
|
+
elif point:
|
|
122
|
+
x, y = point
|
|
123
|
+
zoom_levels = get_zoom_levels(
|
|
124
|
+
process_zoom_levels=_raw_conf()["zoom_levels"],
|
|
125
|
+
init_zoom_levels=zoom
|
|
126
|
+
)
|
|
127
|
+
bounds = _tp().tile_from_xy(x, y, max(zoom_levels)).bounds
|
|
128
|
+
else:
|
|
129
|
+
bounds = bounds
|
|
130
|
+
with mapchete.open(
|
|
131
|
+
mapchete_file, bounds=bounds, zoom=zoom,
|
|
132
|
+
mode=mode, single_input_file=input_file
|
|
133
|
+
) as mp:
|
|
134
|
+
spinner.stop()
|
|
135
|
+
tiles_count = mp.count_tiles(
|
|
136
|
+
min(mp.config.init_zoom_levels),
|
|
137
|
+
max(mp.config.init_zoom_levels))
|
|
138
|
+
tqdm.tqdm.write(
|
|
139
|
+
"processing %s tile(s) on %s worker(s)" % (tiles_count, multi)
|
|
140
|
+
)
|
|
141
|
+
for process_info in tqdm.tqdm(
|
|
142
|
+
mp.batch_processor(
|
|
143
|
+
multi=multi, zoom=zoom,
|
|
144
|
+
max_chunksize=max_chunksize),
|
|
145
|
+
total=tiles_count,
|
|
146
|
+
unit="tile",
|
|
147
|
+
disable=debug or no_pbar
|
|
148
|
+
):
|
|
149
|
+
utils.write_verbose_msg(process_info, dst=verbose_dst)
|
|
150
|
+
|
|
151
|
+
if vrt:
|
|
152
|
+
tqdm.tqdm.write("creating VRT(s)")
|
|
153
|
+
for z in mp.config.init_zoom_levels:
|
|
154
|
+
logger.debug("zoom %s", z)
|
|
155
|
+
for tile in tqdm.tqdm(
|
|
156
|
+
zoom_index_gen(
|
|
157
|
+
mp=mp,
|
|
158
|
+
zoom=z,
|
|
159
|
+
out_dir=(
|
|
160
|
+
idx_out_dir if idx_out_dir
|
|
161
|
+
else mp.config.output.path
|
|
162
|
+
),
|
|
163
|
+
vrt=vrt
|
|
164
|
+
),
|
|
165
|
+
total=mp.count_tiles(z, z),
|
|
166
|
+
unit="tile",
|
|
167
|
+
disable=debug
|
|
168
|
+
):
|
|
169
|
+
logger.debug(tile)
|
|
170
|
+
|
|
171
|
+
tqdm.tqdm.write("process finished", file=verbose_dst)
|