MapProxy 2.1.0__py3-none-any.whl

mapproxy/source/mapnik.py

@@ -0,0 +1,262 @@
+# This file is part of the MapProxy project.
+# Copyright (C) 2011 Omniscale <http://omniscale.de>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+import logging
+
+import sys
+import time
+import threading
+import multiprocessing
+from io import BytesIO
+
+from mapproxy.grid import tile_grid
+from mapproxy.image import ImageSource
+from mapproxy.image.opts import ImageOptions
+from mapproxy.layer import MapExtent, DefaultMapExtent, BlankImage, MapLayer
+from mapproxy.source import SourceError
+from mapproxy.client.log import log_request
+from mapproxy.util.py import reraise_exception
+from mapproxy.util.async_ import run_non_blocking
+
+try:
+    import mapnik
+    mapnik
+except ImportError:
+    try:
+        # for 2.0 alpha/rcs and first 2.0 release
+        import mapnik2 as mapnik
+    except ImportError:
+        mapnik = None
+
+try:
+    import queue
+    Queue = queue.Queue
+    Empty = queue.Empty
+    Full = queue.Full
+except ImportError:  # in python2 it is called Queue
+    import Queue
+    Empty = Queue.Empty
+    Full = Queue.Full
+MAX_UNUSED_MAPS = 10
+
+# fake 2.0 API for older versions
+if mapnik and not hasattr(mapnik, 'Box2d'):
+    mapnik.Box2d = mapnik.Envelope
+
+log = logging.getLogger(__package__)
+
+
+class MapnikSource(MapLayer):
+    supports_meta_tiles = True
+
+    def __init__(self, mapfile, layers=None, image_opts=None, coverage=None,
+                 res_range=None, lock=None, reuse_map_objects=False,
+                 scale_factor=None, multithreaded=False):
+        MapLayer.__init__(self, image_opts=image_opts)
+        self.mapfile = mapfile
+        self.coverage = coverage
+        self.res_range = res_range
+        self.layers = set(layers) if layers else None
+        self.scale_factor = scale_factor
+        self.lock = lock
+        self.multithreaded = multithreaded
+        self._cache_map_obj = reuse_map_objects
+        if self.coverage:
+            self.extent = MapExtent(self.coverage.bbox, self.coverage.srs)
+        else:
+            self.extent = DefaultMapExtent()
+        if multithreaded:
+            # global objects allow caching over multiple instances within the same worker process
+            global _map_objs  # mapnik map objects by cachekey
+            _map_objs = {}
+            global _map_objs_lock
+            _map_objs_lock = threading.Lock()
+            global _map_objs_queues  # queues of unused mapnik map objects by PID and mapfile
+            _map_objs_queues = {}
+        else:
+            # instance variables guarantee separation of caches
+            self._map_objs = {}
+            self._map_objs_lock = threading.Lock()
+
+    def _map_cache(self):
+        """Get the cache for map objects.
+
+        Uses an instance variable for containment when multithreaded
+        operation is disabled.
+        """
+        if self.multithreaded:
+            return _map_objs
+        return self._map_objs
+
+    def _map_cache_lock(self):
+        """Get the cache-locks for map objects.
+
+        Uses an instance variable for containment when multithreaded
+        operation is disabled.
+        """
+        if self.multithreaded:
+            return _map_objs_lock
+        return self._map_objs_lock
+
+    def get_map(self, query):
+        if self.res_range and not self.res_range.contains(query.bbox, query.size,
+                                                          query.srs):
+            raise BlankImage()
+        if self.coverage and not self.coverage.intersects(query.bbox, query.srs):
+            raise BlankImage()
+
+        try:
+            resp = self.render(query)
+        except RuntimeError as ex:
+            log.error('could not render Mapnik map: %s', ex)
+            reraise_exception(SourceError(ex.args[0]), sys.exc_info())
+        resp.opacity = self.opacity
+        return resp
+
+    def render(self, query):
+        mapfile = self.mapfile
+        if '%(webmercator_level)' in mapfile:
+            _bbox, level = tile_grid(3857).get_affected_bbox_and_level(
+                query.bbox, query.size, req_srs=query.srs)
+            mapfile = mapfile % {'webmercator_level': level}
+
+        if self.lock:
+            with self.lock():
+                return self.render_mapfile(mapfile, query)
+        else:
+            return self.render_mapfile(mapfile, query)
+
+    def _create_map_obj(self, mapfile, process_id):
+        m = mapnik.Map(0, 0)
+        mapnik.load_map(m, str(mapfile))
+        m.map_obj_pid = process_id
+        return m
+
+    def _get_map_obj(self, mapfile):
+        if not self.multithreaded:
+            return self._create_map_obj(mapfile, None)
+
+        process_id = multiprocessing.current_process()._identity
+        queue_cachekey = (process_id, mapfile)
+        if queue_cachekey in _map_objs_queues:
+            try:
+                m = _map_objs_queues[queue_cachekey].get_nowait()
+                # check explicitly for the process ID to ensure that
+                # map objects cannot move between processes
+                if m.map_obj_pid == process_id:
+                    return m
+            except Empty:
+                pass
+        return self._create_map_obj(mapfile, process_id)
+
+    def _put_unused_map_obj(self, mapfile, m):
+        process_id = multiprocessing.current_process()._identity
+        queue_cachekey = (process_id, mapfile)
+        if queue_cachekey not in _map_objs_queues:
+            _map_objs_queues[queue_cachekey] = Queue(MAX_UNUSED_MAPS)
+        try:
+            _map_objs_queues[queue_cachekey].put_nowait(m)
+        except Full:
+            # cleanup the data and drop the map, so it can be garbage collected
+            m.remove_all()
+            mapnik.clear_cache()
+
+    def _get_cachekey(self, mapfile):
+        if not self.multithreaded or self._cache_map_obj:
+            # all MapnikSources with the same mapfile share the same Mapnik Map.
+            return (None, None, mapfile)
+        thread_id = threading.current_thread().ident
+        process_id = multiprocessing.current_process()._identity
+        return (process_id, thread_id, mapfile)
+
+    def _cleanup_unused_cached_maps(self, mapfile):
+        if not self.multithreaded:
+            return
+        # clean up no longer used cached maps
+        process_id = multiprocessing.current_process()._identity
+        process_cache_keys = [k for k in self._map_cache().keys()
+                              if k[0] == process_id]
+        # To avoid time-consuming cleanup whenever one thread in the
+        # threadpool finishes, allow ignoring up to 5 dead mapnik
+        # instances.  (5 is empirical)
+        if len(process_cache_keys) > (5 + threading.active_count()):
+            active_thread_ids = set(i.ident for i in threading.enumerate())
+            for k in process_cache_keys:
+                with self._map_cache_lock():
+                    if not k[1] in active_thread_ids and k in self._map_cache():
+                        try:
+                            m = self._map_cache()[k]
+                            del self._map_cache()[k]
+                            # put the map into the queue of unused
+                            # maps so it can be re-used from another
+                            # thread.
+                            self._put_unused_map_obj(mapfile, m)
+                        except KeyError:
+                            continue
+
+    def map_obj(self, mapfile):
+        # cache loaded map objects
+        # only works when a single proc/thread accesses the map
+        # (forking the render process doesn't work because of open database
+        #  file handles that gets passed to the child)
+        # segment the cache by process and thread to avoid interference
+        cachekey = self._get_cachekey(mapfile)
+        with self._map_cache_lock():
+            if cachekey not in self._map_cache():
+                self._map_cache()[cachekey] = self._get_map_obj(mapfile)
+            mapnik_map = self._map_cache()[cachekey]
+
+        self._cleanup_unused_cached_maps(mapfile)
+
+        return mapnik_map
+
+    def render_mapfile(self, mapfile, query):
+        return run_non_blocking(self._render_mapfile, (mapfile, query))
+
+    def _render_mapfile(self, mapfile, query):
+        start_time = time.time()
+
+        m = self.map_obj(mapfile)
+        m.resize(query.size[0], query.size[1])
+        m.srs = '+init=%s' % str(query.srs.srs_code.lower())
+        envelope = mapnik.Box2d(*query.bbox)
+        m.zoom_to_box(envelope)
+        data = None
+
+        try:
+            if self.layers:
+                i = 0
+                for layer in m.layers[:]:
+                    if layer.name != 'Unknown' and layer.name not in self.layers:
+                        del m.layers[i]
+                    else:
+                        i += 1
+
+            img = mapnik.Image(query.size[0], query.size[1])
+            if self.scale_factor:
+                mapnik.render(m, img, self.scale_factor)
+            else:
+                mapnik.render(m, img)
+            data = img.tostring(str(query.format))
+        finally:
+            size = None
+            if data:
+                size = len(data)
+            log_request('%s:%s:%s:%s' % (mapfile, query.bbox, query.srs.srs_code, query.size),
+                        status='200' if data else '500', size=size, method='API', duration=time.time()-start_time)
+
+        return ImageSource(BytesIO(data), size=query.size,
+                           image_opts=ImageOptions(format=query.format))

mapproxy/source/tile.py

@@ -0,0 +1,97 @@
+# This file is part of the MapProxy project.
+# Copyright (C) 2010 Omniscale <http://omniscale.de>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Retrieve tiles from different tile servers (TMS/TileCache/etc.).
+"""
+
+import sys
+from mapproxy.image.opts import ImageOptions
+from mapproxy.source import SourceError
+from mapproxy.client.http import HTTPClientError
+from mapproxy.source import InvalidSourceQuery
+from mapproxy.layer import BlankImage, map_extent_from_grid, CacheMapLayer, MapLayer
+from mapproxy.util.py import reraise_exception
+
+import logging
+log = logging.getLogger('mapproxy.source.tile')
+log_config = logging.getLogger('mapproxy.config')
+
+
+class TiledSource(MapLayer):
+    def __init__(self, grid, client, coverage=None, image_opts=None, error_handler=None,
+                 res_range=None):
+        MapLayer.__init__(self, image_opts=image_opts)
+        self.grid = grid
+        self.client = client
+        self.image_opts = image_opts or ImageOptions()
+        self.coverage = coverage
+        self.extent = coverage.extent if coverage else map_extent_from_grid(grid)
+        self.res_range = res_range
+        self.error_handler = error_handler
+
+    def get_map(self, query):
+        if self.grid.tile_size != query.size:
+            ex = InvalidSourceQuery(
+                'tile size of cache and tile source do not match: %s != %s'
+                % (self.grid.tile_size, query.size)
+            )
+            log_config.error(ex)
+            raise ex
+
+        if self.grid.srs != query.srs:
+            ex = InvalidSourceQuery(
+                'SRS of cache and tile source do not match: %r != %r'
+                % (self.grid.srs, query.srs)
+            )
+            log_config.error(ex)
+            raise ex
+
+        if self.res_range and not self.res_range.contains(query.bbox, query.size,
+                                                          query.srs):
+            raise BlankImage()
+        if self.coverage and not self.coverage.intersects(query.bbox, query.srs):
+            raise BlankImage()
+
+        _bbox, grid, tiles = self.grid.get_affected_tiles(query.bbox, query.size)
+
+        if grid != (1, 1):
+            raise InvalidSourceQuery('BBOX does not align to tile')
+
+        tile_coord = next(tiles)
+
+        try:
+            return self.client.get_tile(tile_coord, format=query.format)
+        except HTTPClientError as e:
+            if self.error_handler:
+                resp = self.error_handler.handle(e.response_code, query)
+                if resp:
+                    return resp
+            log.warning('could not retrieve tile: %s', e)
+            reraise_exception(SourceError(e.args[0]), sys.exc_info())
+
+
+class CacheSource(CacheMapLayer):
+    def __init__(self, tile_manager, extent=None, image_opts=None,
+                 max_tile_limit=None, tiled_only=False):
+        CacheMapLayer.__init__(self, tile_manager, extent=extent, image_opts=image_opts,
+                               max_tile_limit=max_tile_limit)
+        self.supports_meta_tiles = not tiled_only
+        self.tiled_only = tiled_only
+
+    def get_map(self, query):
+        if self.tiled_only:
+            query.tiled_only = True
+        return CacheMapLayer.get_map(self, query)

mapproxy/source/wms.py

@@ -0,0 +1,273 @@
+# This file is part of the MapProxy project.
+# Copyright (C) 2010 Omniscale <http://omniscale.de>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Retrieve maps/information from WMS servers.
+"""
+import sys
+from mapproxy.request.base import split_mime_type
+from mapproxy.cache.legend import Legend, legend_identifier
+from mapproxy.image import make_transparent, ImageSource, SubImageSource, bbox_position_in_image
+from mapproxy.image.merge import concat_legends, concat_json_legends
+from mapproxy.image.transform import ImageTransformer
+from mapproxy.layer import MapExtent, DefaultMapExtent, BlankImage, LegendQuery, MapQuery, MapLayer
+from mapproxy.source import InfoSource, SourceError, LegendSource
+from mapproxy.client.http import HTTPClientError
+from mapproxy.util.py import reraise_exception
+
+import logging
+log = logging.getLogger('mapproxy.source.wms')
+
+
+class WMSSource(MapLayer):
+    supports_meta_tiles = True
+
+    def __init__(self, client, image_opts=None, coverage=None, res_range=None,
+                 transparent_color=None, transparent_color_tolerance=None,
+                 supported_srs=None, supported_formats=None, fwd_req_params=None,
+                 error_handler=None):
+        MapLayer.__init__(self, image_opts=image_opts)
+        self.client = client
+        self.supported_srs = supported_srs or []
+        self.supported_formats = supported_formats or []
+        self.fwd_req_params = fwd_req_params or set()
+
+        self.transparent_color = transparent_color
+        self.transparent_color_tolerance = transparent_color_tolerance
+        if self.transparent_color:
+            self.image_opts.transparent = True
+        self.coverage = coverage
+        self.res_range = res_range
+        if self.coverage:
+            self.extent = MapExtent(self.coverage.bbox, self.coverage.srs)
+        else:
+            self.extent = DefaultMapExtent()
+        self.error_handler = error_handler
+
+    def is_opaque(self, query):
+        """
+        Returns true if we are sure that the image is not transparent.
+        """
+        if self.res_range and not self.res_range.contains(query.bbox, query.size,
+                                                          query.srs):
+            return False
+
+        if self.image_opts.transparent:
+            return False
+
+        if self.opacity is not None and (0.0 < self.opacity < 0.99):
+            return False
+
+        if not self.coverage:
+            # not transparent and no coverage
+            return True
+
+        if self.coverage.contains(query.bbox, query.srs):
+            # not transparent and completely inside coverage
+            return True
+
+        return False
+
+    def get_map(self, query):
+        if self.res_range and not self.res_range.contains(query.bbox, query.size,
+                                                          query.srs):
+            raise BlankImage()
+        if self.coverage and not self.coverage.intersects(query.bbox, query.srs):
+            raise BlankImage()
+        try:
+            resp = self._get_map(query)
+            if self.transparent_color:
+                resp = make_transparent(resp, self.transparent_color,
+                                        self.transparent_color_tolerance)
+            resp.opacity = self.opacity
+            return resp
+
+        except HTTPClientError as e:
+            if self.error_handler:
+                resp = self.error_handler.handle(e.response_code, query)
+                if resp:
+                    return resp
+            log.warning('could not retrieve WMS map: %s', e.full_msg or e)
+            reraise_exception(SourceError(e.args[0]), sys.exc_info())
+
+    def _get_map(self, query):
+        format = self.image_opts.format
+        if not format:
+            format = query.format
+        if self.supported_formats and format not in self.supported_formats:
+            format = self.supported_formats[0]
+        if self.supported_srs:
+            # srs can be equal while still having a different srs_code (EPSG:3857/900913),
+            # make sure to use a supported srs_code
+            request_srs = None
+            for srs in self.supported_srs:
+                if query.srs == srs:
+                    request_srs = srs
+                    break
+            if request_srs is None:
+                return self._get_transformed(query, format)
+            if query.srs.srs_code != request_srs.srs_code:
+                query.srs = request_srs
+        if self.extent and not self.extent.contains(MapExtent(query.bbox, query.srs)):
+            return self._get_sub_query(query, format)
+        resp = self.client.retrieve(query, format)
+        return ImageSource(resp, size=query.size, image_opts=self.image_opts)
+
+    def _get_sub_query(self, query, format):
+        size, offset, bbox = bbox_position_in_image(query.bbox, query.size, self.extent.bbox_for(query.srs))
+        if size[0] == 0 or size[1] == 0:
+            raise BlankImage()
+        src_query = MapQuery(bbox, size, query.srs, format, dimensions=query.dimensions)
+        resp = self.client.retrieve(src_query, format)
+        return SubImageSource(resp, size=query.size, offset=offset, image_opts=self.image_opts)
+
+    def _get_transformed(self, query, format):
+        dst_srs = query.srs
+        src_srs = self.supported_srs.best_srs(dst_srs)
+        dst_bbox = query.bbox
+        src_bbox = dst_srs.transform_bbox_to(src_srs, dst_bbox)
+
+        src_width, src_height = src_bbox[2]-src_bbox[0], src_bbox[3]-src_bbox[1]
+        ratio = src_width/src_height
+        dst_size = query.size
+        xres, yres = src_width/dst_size[0], src_height/dst_size[1]
+        if xres < yres:
+            src_size = dst_size[0], int(dst_size[0]/ratio + 0.5)
+        else:
+            src_size = int(dst_size[1]*ratio + 0.5), dst_size[1]
+
+        src_query = MapQuery(src_bbox, src_size, src_srs, format, dimensions=query.dimensions)
+
+        if self.coverage and not self.coverage.contains(src_bbox, src_srs):
+            img = self._get_sub_query(src_query, format)
+        else:
+            resp = self.client.retrieve(src_query, format)
+            img = ImageSource(resp, size=src_size, image_opts=self.image_opts)
+
+        img = ImageTransformer(src_srs, dst_srs).transform(img, src_bbox,
+                                                           query.size, dst_bbox, self.image_opts)
+
+        img.format = format
+        return img
+
+    def _is_compatible(self, other, query):
+        if not isinstance(other, WMSSource):
+            return False
+
+        if self.opacity is not None or other.opacity is not None:
+            return False
+
+        if self.supported_srs != other.supported_srs:
+            return False
+
+        if self.supported_formats != other.supported_formats:
+            return False
+
+        if self.transparent_color != other.transparent_color:
+            return False
+
+        if self.transparent_color_tolerance != other.transparent_color_tolerance:
+            return False
+
+        if self.coverage != other.coverage:
+            return False
+
+        if (query.dimensions_for_params(self.fwd_req_params) !=
+                query.dimensions_for_params(other.fwd_req_params)):
+            return False
+
+        return True
+
+    def combined_layer(self, other, query):
+        if not self._is_compatible(other, query):
+            return None
+
+        client = self.client.combined_client(other.client, query)
+        if not client:
+            return None
+
+        return WMSSource(client, image_opts=self.image_opts,
+                         transparent_color=self.transparent_color,
+                         transparent_color_tolerance=self.transparent_color_tolerance,
+                         supported_srs=self.supported_srs,
+                         supported_formats=self.supported_formats,
+                         res_range=None,  # layer outside res_range should already be filtered out
+                         coverage=self.coverage,
+                         fwd_req_params=self.fwd_req_params,
+                         )
+
+
+class WMSInfoSource(InfoSource):
+    def __init__(self, client, fi_transformer=None, coverage=None):
+        self.client = client
+        self.fi_transformer = fi_transformer
+        self.coverage = coverage
+
+    def get_info(self, query):
+        if self.coverage and not self.coverage.contains(query.coord, query.srs):
+            return None
+        doc = self.client.get_info(query)
+        if self.fi_transformer:
+            doc = self.fi_transformer(doc)
+        return doc
+
+
+class WMSLegendSource(LegendSource):
+    def __init__(self, clients, legend_cache, static=False):
+        self.clients = clients
+        self.identifier = legend_identifier([c.identifier for c in self.clients])
+        self._cache = legend_cache
+        self._size = None
+        self.static = static
+
+    @property
+    def size(self):
+        if not self._size:
+            legend = self.get_legend(LegendQuery(format='image/png', scale=None))
+            # TODO image size without as_image?
+            self._size = legend.as_image().size
+        return self._size
+
+    def get_legend(self, query):
+        if self.static:
+            # prevent caching of static legends for different scales
+            legend = Legend(id=self.identifier, scale=None)
+        else:
+            legend = Legend(id=self.identifier, scale=query.scale)
+
+        if not self._cache.load(legend) or query.format == 'json':
+            legends = []
+            error_occured = False
+            for client in self.clients:
+                try:
+                    legends.append(client.get_legend(query))
+                except HTTPClientError as e:
+                    error_occured = True
+                    log.error(e.args[0])
+                except SourceError as e:
+                    error_occured = True
+                    # TODO errors?
+                    log.error(e.args[0])
+            format = split_mime_type(query.format)[1]
+
+            if format == 'json':
+                return concat_json_legends(legends)
+
+            legend = Legend(source=concat_legends(legends, format=format),
+                            id=self.identifier, scale=query.scale)
+            if not error_occured:
+                self._cache.store(legend)
+
+        # returns ImageSource
+        return legend.source